From 44191daf727be0356d1fef6ca1240c22a0bb04c4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:55:50 +0100 Subject: [PATCH 01/71] feat: get changes from the PoC (platform secrets as sealed secrest) branches --- chart/apl/templates/deployment.yaml | 4 - charts/external-secrets/Chart.yaml | 16 + .../crds/clusterexternalsecrets.yaml | 51 ++ .../crds/clustersecretstores.yaml | 48 ++ .../crds/externalsecrets.yaml | 51 ++ .../crds/generatorstates.yaml | 39 ++ charts/external-secrets/crds/pushsecrets.yaml | 43 ++ .../external-secrets/crds/secretstores.yaml | 48 ++ .../external-secrets/templates/_helpers.tpl | 41 ++ .../templates/clusterrole.yaml | 37 ++ .../templates/deployment.yaml | 33 + .../templates/serviceaccount.yaml | 9 + charts/external-secrets/values.yaml | 31 + .../ingress-nginx/templates/clusterrole.yaml | 1 + charts/otomi-api/templates/deployment.yaml | 8 + charts/otomi-api/values.yaml | 1 - helmfile.d/helmfile-01.init.yaml.gotmpl | 8 + helmfile.d/helmfile-04.init.yaml.gotmpl | 6 + helmfile.d/helmfile-60.teams.yaml.gotmpl | 83 ++- helmfile.d/helmfile-70.shared.yaml.gotmpl | 7 + helmfile.d/snippets/alertmanager-teams.gotmpl | 8 +- helmfile.d/snippets/alertmanager.gotmpl | 8 +- helmfile.d/snippets/defaults.gotmpl | 10 +- helmfile.d/snippets/defaults.yaml | 10 + helmfile.d/snippets/derived.gotmpl | 7 +- helmfile.d/snippets/grafana.gotmpl | 4 +- helmfile.d/snippets/sops-env.gotmpl | 24 - package-lock.json | 37 +- src/cmd/bootstrap.test.ts | 85 ++- src/cmd/bootstrap.ts | 54 +- src/cmd/commit.ts | 50 +- src/cmd/install.test.ts | 35 +- src/cmd/install.ts | 154 ++++- src/cmd/migrate.ts | 9 +- src/cmd/validate-values.ts | 53 +- src/common/bootstrap.ts | 13 +- src/common/git-config.test.ts | 76 ++- src/common/git-config.ts | 26 +- src/common/repo.ts | 1 - src/common/sealed-secrets.test.ts | 575 ++++++++++++++++ src/common/sealed-secrets.ts | 628 ++++++++++++++++++ src/common/values.ts | 29 +- src/operator/installer.test.ts | 72 +- src/operator/installer.ts | 41 +- values-schema.yaml | 13 +- .../apl-gitea-operator-raw.gotmpl | 33 +- .../apl-harbor-operator-raw.gotmpl | 35 +- .../apl-keycloak-operator-raw.gotmpl | 65 +- values/apl-operator/apl-operator-raw.gotmpl | 25 + values/apl-operator/apl-operator.gotmpl | 9 - values/argocd/argocd-raw.gotmpl | 102 ++- values/argocd/argocd.gotmpl | 3 +- values/cert-manager/cert-manager-raw.gotmpl | 139 +++- values/external-dns/external-dns-raw.gotmpl | 241 +++++-- values/external-dns/external-dns.gotmpl | 10 +- .../external-secrets-raw.gotmpl | 47 ++ .../external-secrets/external-secrets.gotmpl | 9 + .../gitea-db-secret-raw.gotmpl | 27 +- values/gitea/gitea-raw.gotmpl | 77 ++- values/gitea/gitea.gotmpl | 9 +- values/harbor/harbor-raw.gotmpl | 217 ++++-- values/harbor/harbor.gotmpl | 3 +- values/ingress-nginx/ingress-nginx-raw.gotmpl | 34 +- values/k8s/k8s-raw.gotmpl | 9 - values/keycloak/keycloak-raw.gotmpl | 52 +- values/loki/loki-raw.gotmpl | 65 +- values/oauth2-proxy/oauth2-proxy-raw.gotmpl | 50 +- values/oauth2-proxy/oauth2-proxy.gotmpl | 5 +- values/otomi-api/otomi-api-raw.gotmpl | 24 + values/otomi-api/otomi-api.gotmpl | 5 +- .../prometheus-operator-raw.gotmpl | 138 +++- .../prometheus-operator.gotmpl | 27 +- versions.yaml | 2 +- 73 files changed, 3594 insertions(+), 455 deletions(-) create mode 100644 charts/external-secrets/Chart.yaml create mode 100644 charts/external-secrets/crds/clusterexternalsecrets.yaml create mode 100644 charts/external-secrets/crds/clustersecretstores.yaml create mode 100644 charts/external-secrets/crds/externalsecrets.yaml create mode 100644 charts/external-secrets/crds/generatorstates.yaml create mode 100644 charts/external-secrets/crds/pushsecrets.yaml create mode 100644 charts/external-secrets/crds/secretstores.yaml create mode 100644 charts/external-secrets/templates/_helpers.tpl create mode 100644 charts/external-secrets/templates/clusterrole.yaml create mode 100644 charts/external-secrets/templates/deployment.yaml create mode 100644 charts/external-secrets/templates/serviceaccount.yaml create mode 100644 charts/external-secrets/values.yaml create mode 100644 src/common/sealed-secrets.test.ts create mode 100644 src/common/sealed-secrets.ts create mode 100644 values/apl-operator/apl-operator-raw.gotmpl create mode 100644 values/external-secrets/external-secrets-raw.gotmpl create mode 100644 values/external-secrets/external-secrets.gotmpl create mode 100644 values/otomi-api/otomi-api-raw.gotmpl diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 7f6bff1de9..0661532ea4 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -75,13 +75,9 @@ spec: value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }} - name: RECONCILE_INTERVAL_MS value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }} - {{- if hasKey $kms "sops" }} envFrom: - - secretRef: - name: apl-sops-secrets - secretRef: name: apl-git-credentials - {{- end }} volumeMounts: - name: otomi-values mountPath: /home/app/stack/env diff --git a/charts/external-secrets/Chart.yaml b/charts/external-secrets/Chart.yaml new file mode 100644 index 0000000000..f166cf473d --- /dev/null +++ b/charts/external-secrets/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +appVersion: 0.14.3 +description: External Secrets Operator for Kubernetes +home: https://external-secrets.io +keywords: +- secrets +- external-secrets +kubeVersion: '>=1.19.0-0' +maintainers: +- name: External Secrets Community + url: https://github.com/external-secrets/external-secrets +name: external-secrets +sources: +- https://github.com/external-secrets/external-secrets +type: application +version: 0.14.3 diff --git a/charts/external-secrets/crds/clusterexternalsecrets.yaml b/charts/external-secrets/crds/clusterexternalsecrets.yaml new file mode 100644 index 0000000000..bda5779fe9 --- /dev/null +++ b/charts/external-secrets/crds/clusterexternalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterexternalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterExternalSecret + listKind: ClusterExternalSecretList + plural: clusterexternalsecrets + shortNames: + - ces + singular: clusterexternalsecret + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.externalSecretSpec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshTime + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterExternalSecret creates ExternalSecrets across namespaces + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/clustersecretstores.yaml b/charts/external-secrets/crds/clustersecretstores.yaml new file mode 100644 index 0000000000..568ca452f1 --- /dev/null +++ b/charts/external-secrets/crds/clustersecretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustersecretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterSecretStore + listKind: ClusterSecretStoreList + plural: clustersecretstores + shortNames: + - css + singular: clustersecretstore + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterSecretStore represents a cluster-wide secret store + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/externalsecrets.yaml b/charts/external-secrets/crds/externalsecrets.yaml new file mode 100644 index 0000000000..07ba80b4e5 --- /dev/null +++ b/charts/external-secrets/crds/externalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ExternalSecret + listKind: ExternalSecretList + plural: externalsecrets + shortNames: + - es + singular: externalsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshInterval + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ExternalSecret reads secret data from external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/generatorstates.yaml b/charts/external-secrets/crds/generatorstates.yaml new file mode 100644 index 0000000000..0f50ec67aa --- /dev/null +++ b/charts/external-secrets/crds/generatorstates.yaml @@ -0,0 +1,39 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: generatorstates.generators.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: generators.external-secrets.io + names: + categories: + - externalsecrets + kind: GeneratorState + listKind: GeneratorStateList + plural: generatorstates + singular: generatorstate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GeneratorState tracks the state of generators + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/pushsecrets.yaml b/charts/external-secrets/crds/pushsecrets.yaml new file mode 100644 index 0000000000..7595969ca1 --- /dev/null +++ b/charts/external-secrets/crds/pushsecrets.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: pushsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: PushSecret + listKind: PushSecretList + plural: pushsecrets + singular: pushsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: PushSecret pushes secrets to external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/secretstores.yaml b/charts/external-secrets/crds/secretstores.yaml new file mode 100644 index 0000000000..a61ca3b805 --- /dev/null +++ b/charts/external-secrets/crds/secretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: secretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: SecretStore + listKind: SecretStoreList + plural: secretstores + shortNames: + - ss + singular: secretstore + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: SecretStore represents a source of secrets + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/templates/_helpers.tpl b/charts/external-secrets/templates/_helpers.tpl new file mode 100644 index 0000000000..d090bbc336 --- /dev/null +++ b/charts/external-secrets/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "external-secrets.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "external-secrets.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "external-secrets.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "external-secrets.selectorLabels" -}} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/external-secrets/templates/clusterrole.yaml b/charts/external-secrets/templates/clusterrole.yaml new file mode 100644 index 0000000000..daee3d4cd9 --- /dev/null +++ b/charts/external-secrets/templates/clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +rules: + - apiGroups: ["external-secrets.io"] + resources: ["externalsecrets", "externalsecrets/status", "externalsecrets/finalizers", "secretstores", "secretstores/status", "secretstores/finalizers", "clustersecretstores", "clustersecretstores/status", "clustersecretstores/finalizers", "clusterexternalsecrets", "clusterexternalsecrets/status", "clusterexternalsecrets/finalizers", "pushsecrets", "pushsecrets/status", "pushsecrets/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: ["generators.external-secrets.io"] + resources: ["generatorstates", "generatorstates/status", "generatorstates/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["secrets", "configmaps", "serviceaccounts", "serviceaccounts/token", "events"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "external-secrets.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} diff --git a/charts/external-secrets/templates/deployment.yaml b/charts/external-secrets/templates/deployment.yaml new file mode 100644 index 0000000000..2b2b4ea088 --- /dev/null +++ b/charts/external-secrets/templates/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "external-secrets.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "external-secrets.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "external-secrets.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + containers: + - name: external-secrets + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --concurrent=1 + - --metrics-addr=:8080 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/charts/external-secrets/templates/serviceaccount.yaml b/charts/external-secrets/templates/serviceaccount.yaml new file mode 100644 index 0000000000..c12da9edf8 --- /dev/null +++ b/charts/external-secrets/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/external-secrets/values.yaml b/charts/external-secrets/values.yaml new file mode 100644 index 0000000000..5f2abea1d5 --- /dev/null +++ b/charts/external-secrets/values.yaml @@ -0,0 +1,31 @@ +## External Secrets Operator values + +## @param replicaCount Number of ESO controller replicas +replicaCount: 1 + +## @param image ESO controller image configuration +image: + repository: ghcr.io/external-secrets/external-secrets + tag: v0.14.3 + pullPolicy: IfNotPresent + +## @param serviceAccount Service account configuration +serviceAccount: + create: true + name: external-secrets + +## @param installCRDs Install ESO CRDs +installCRDs: true + +## @param resources Resource limits and requests +resources: {} + +## @param webhook Webhook configuration +webhook: + replicaCount: 1 + resources: {} + +## @param certController Cert controller configuration +certController: + replicaCount: 1 + resources: {} diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 51bc5002cc..15b2ac4b55 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,6 +27,7 @@ rules: - namespaces {{- end}} verbs: + - get - list - watch - apiGroups: diff --git a/charts/otomi-api/templates/deployment.yaml b/charts/otomi-api/templates/deployment.yaml index 527ca0a1bc..b4ba47d821 100644 --- a/charts/otomi-api/templates/deployment.yaml +++ b/charts/otomi-api/templates/deployment.yaml @@ -38,6 +38,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} livenessProbe: @@ -75,6 +79,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} ports: diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index 8c7793b4fe..8cdfe5787a 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -87,7 +87,6 @@ affinity: {} secrets: GIT_USER: GIT_EMAIL: - GIT_PASSWORD: env: GIT_REPO_URL: diff --git a/helmfile.d/helmfile-01.init.yaml.gotmpl b/helmfile.d/helmfile-01.init.yaml.gotmpl index 6287a13d27..c31194a595 100644 --- a/helmfile.d/helmfile-01.init.yaml.gotmpl +++ b/helmfile.d/helmfile-01.init.yaml.gotmpl @@ -34,6 +34,14 @@ releases: installed: {{ $a | get "sealed-secrets.enabled" }} namespace: sealed-secrets <<: *default + - name: external-secrets + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *default + - name: external-secrets-artifacts + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *raw - name: cert-manager installed: true namespace: cert-manager diff --git a/helmfile.d/helmfile-04.init.yaml.gotmpl b/helmfile.d/helmfile-04.init.yaml.gotmpl index 2e6800ce8e..df2adc1e35 100644 --- a/helmfile.d/helmfile-04.init.yaml.gotmpl +++ b/helmfile.d/helmfile-04.init.yaml.gotmpl @@ -25,6 +25,12 @@ releases: labels: pkg: apl-operator <<: *default + - name: apl-operator-artifacts + installed: true + namespace: apl-operator + labels: + pkg: apl-operator + <<: *raw - name: otomi-operator installed: true namespace: otomi-operator diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index dfcac659cc..ce769d6040 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -24,6 +24,8 @@ releases: {{- $prometheusDomain := printf "prometheus-%s.%s" $teamId $domain }} {{- $grafanaDomain := printf "grafana-%s.%s" $teamId $domain }} {{- $teamApps := index $tc $teamId "apps" | default dict }} + {{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} + {{- $teamAlertmanagerConfig := tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} - name: tekton-dashboard-{{ $teamId }} installed: true namespace: team-{{ $teamId }} @@ -61,8 +63,8 @@ releases: prometheus: system resources: {{- $teamApps.alertmanager.resources | toYaml | nindent 14 }} - # to do: load slackTpl and opsgenieTpl only if alerts.receicers = true - config: {{- tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 12 }} + useExistingSecret: true + configSecret: alertmanager-team-{{ $teamId }}-config defaultRules: rules: general: false @@ -73,7 +75,10 @@ releases: prometheusSpec: {} grafana: enabled: {{ $teamSettings | get "managedMonitoring.grafana" false }} - adminPassword: {{ $teamSettings.password | quote }} + admin: + existingSecret: team-{{ $teamId }}-grafana-admin + userKey: admin-user + passwordKey: admin-password resources: {{- $teamApps.grafana.resources.grafana | toYaml | nindent 12 }} namespaceOverride: null # team-{{ $teamId }} @@ -116,8 +121,6 @@ releases: url: http://loki-query-frontend-headless.monitoring:3101 basicAuth: true basicAuthUser: {{ $teamId }} - secureJsonData: - basicAuthPassword: {{ $teamSettings.password | quote }} {{- if has "msteams" ($teamSettings | get "alerts.receivers" list) }} - name: prometheus-msteams-{{ $teamId }} installed: {{ $teamSettings | get "managedMonitoring.alertmanager" false }} @@ -166,4 +169,74 @@ releases: pipeline: otomi-task-teams values: - ../values/team-ns/team-ns.gotmpl + - name: team-secrets-{{ $teamId }} + installed: true + namespace: team-{{ $teamId }} + chart: ../charts/raw + labels: + tag: teams + team: {{ $teamId }} + pipeline: otomi-task-teams + values: + - resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: team-{{ $teamId }}-grafana-admin + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: team-{{ $teamId }}-grafana-admin + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: {{ $teamId }} + admin-password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: team-{{ $teamId }}-settings-secrets + property: settings_password + {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-team-{{ $teamId }}-config + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-team-{{ $teamId }}-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $teamAlertmanagerConfig | nindent 20 }} + data: + {{- if has "slack" $teamReceivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $teamReceivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- end }} {{- end }} diff --git a/helmfile.d/helmfile-70.shared.yaml.gotmpl b/helmfile.d/helmfile-70.shared.yaml.gotmpl index 2b0f64b250..6b25e08103 100644 --- a/helmfile.d/helmfile-70.shared.yaml.gotmpl +++ b/helmfile.d/helmfile-70.shared.yaml.gotmpl @@ -35,6 +35,13 @@ releases: pkg: oauth2-proxy app: core <<: *raw + - name: otomi-api-artifacts + installed: true + namespace: otomi + labels: + pkg: otomi + app: core + <<: *raw - name: otomi-api installed: true namespace: otomi diff --git a/helmfile.d/snippets/alertmanager-teams.gotmpl b/helmfile.d/snippets/alertmanager-teams.gotmpl index aefb5e30af..5f2a51ba8d 100644 --- a/helmfile.d/snippets/alertmanager-teams.gotmpl +++ b/helmfile.d/snippets/alertmanager-teams.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/alertmanager.gotmpl b/helmfile.d/snippets/alertmanager.gotmpl index a4eb2346bc..24448df09d 100644 --- a/helmfile.d/snippets/alertmanager.gotmpl +++ b/helmfile.d/snippets/alertmanager.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/defaults.gotmpl b/helmfile.d/snippets/defaults.gotmpl index df5cca08c3..e271a1f656 100644 --- a/helmfile.d/snippets/defaults.gotmpl +++ b/helmfile.d/snippets/defaults.gotmpl @@ -22,8 +22,8 @@ environments: default: values: - apps: - kubeflow-pipelines: - rootPassword: {{ randAlphaNum 32 }} + kubeflow-pipelines: {} + gitea: {} {{- range $index,$ingressClassName := $ingressClassNames }} ingress-nginx-{{ $ingressClassName}}: autoscaling: @@ -267,13 +267,9 @@ environments: useCloudShell: true downloadKubeconfig: true downloadDockerLogin: true - password: {{ randAlphaNum 32 }} {{- end }} {{- end }} - otomi: - adminPassword: {{ randAlphaNum 32 }} - git: - password: {{ randAlphaNum 20 }} + otomi: {} cluster: owner: customer name: apl diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index ae92ed7377..1f60127045 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -977,6 +977,16 @@ environments: cpu: "2" memory: 1Gi _rawValues: {} + external-secrets: + resources: + operator: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 512Mi + _rawValues: {} rabbitmq: enabled: false resources: diff --git a/helmfile.d/snippets/derived.gotmpl b/helmfile.d/snippets/derived.gotmpl index 569f30c8b6..0a7c209f99 100644 --- a/helmfile.d/snippets/derived.gotmpl +++ b/helmfile.d/snippets/derived.gotmpl @@ -220,15 +220,10 @@ environments: {{- end }} external-dns: enabled: {{ $v.otomi.hasExternalDNS }} - harbor: - adminPassword: {{ $a | get "harbor.adminPassword" $v.otomi.adminPassword | quote }} - registry: - credentials: - password: {{ $a | get "harbor.registry.credentials.password" $v.otomi.adminPassword | quote }} + harbor: {} keycloak: enabled: true address: {{ $keycloakBaseUrl }} - adminPassword: {{ $a | get "keycloak.adminPassword" $v.otomi.adminPassword | quote }} ingress-nginx: enabled: true istio: diff --git a/helmfile.d/snippets/grafana.gotmpl b/helmfile.d/snippets/grafana.gotmpl index 9128cca891..4490a0f2fd 100644 --- a/helmfile.d/snippets/grafana.gotmpl +++ b/helmfile.d/snippets/grafana.gotmpl @@ -9,8 +9,8 @@ analytics: org_role: Admin allow_sign_up: true oauth_auto_login: true - client_id: {{ .keycloak.clientID }} - client_secret: {{ .keycloak.clientSecret | quote }} + client_id: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_ID} + client_secret: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} scopes: email profile openid email_attribute_path: email login_attribute_path: username diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index aa9e2ff866..e69de29bb2 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -1,24 +0,0 @@ -{{- with . | get "azure" nil }} -AZURE_CLIENT_ID: {{ .clientId }} -AZURE_CLIENT_SECRET: {{ .clientSecret }} -{{- with . | get "tenantId" nil }} -AZURE_TENANT_ID: {{ . }}{{ end }} -{{- with . | get "environment" nil }} -AZURE_ENVIRONMENT: {{ . }}{{ end }} -{{- end }} -{{- with . | get "aws" nil }} -AWS_ACCESS_KEY_ID: {{ .accessKey }} -AWS_SECRET_ACCESS_KEY: {{ .secretKey }} -{{- with . | get "region" nil }} -AWS_REGION: {{ . }}{{ end }} -{{- end }} -{{- with . | get "age" nil }} -SOPS_AGE_KEY: {{ .privateKey }} -{{- end }} -{{- with . | get "google" nil }} -GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' -{{- with . | get "project" nil }} -GOOGLE_PROJECT: {{ . }}{{ end }} -{{- with . | get "region" nil }} -GOOGLE_REGION: {{ . }}{{ end }} -{{- end }} diff --git a/package-lock.json b/package-lock.json index e1ace274da..8bdaab5dfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -200,6 +200,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2465,6 +2466,7 @@ "integrity": "sha512-lf6d+BdMkJIFCxx2FpajLpqVGGyaGUNFU6jhEM6QUPeGuoA5et2kJXrL0NSY2uWLOVyYYc/FPjzlbe8trA9tBQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -2546,7 +2548,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.19.tgz", "integrity": "sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2686,14 +2689,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.14.tgz", "integrity": "sha512-2bf7n+kS92g+cMKV0wr9o/Oq9n8JzU7CcrB96gIh2GHgnF+0xDOqO2W/1KeFAqOfqosoOVE48t+4dnEMkkoJ2Q==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -2891,7 +2896,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4983,6 +4989,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6572,7 +6579,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6638,6 +6646,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6837,6 +6846,7 @@ "integrity": "sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.55.0", "@typescript-eslint/types": "8.55.0", @@ -7377,6 +7387,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8258,6 +8269,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10655,6 +10667,7 @@ "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -12049,6 +12062,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -12109,6 +12123,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -12243,6 +12258,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -15752,6 +15768,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -17519,6 +17536,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -18562,6 +18580,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -21666,6 +21685,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22721,6 +22741,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -23722,6 +23743,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -25582,6 +25604,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -25785,6 +25808,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -26082,6 +26106,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -26261,6 +26286,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.2.4" }, @@ -26763,6 +26789,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 1417769ca7..6cfb0f22a1 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -12,6 +12,8 @@ import { processValues, } from './bootstrap' +jest.mock('@linode/kubeseal-encrypt') + const { terminal } = stubs jest.mock('src/common/envalid', () => ({ @@ -37,6 +39,7 @@ describe('Bootstrapping values', () => { }), }), bootstrapSops: jest.fn(), + bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), @@ -59,14 +62,14 @@ describe('Bootstrapping values', () => { } }) it('should call relevant sub routines', async () => { - deps.processValues.mockReturnValue(values) + deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() - expect(deps.bootstrapSops).toHaveBeenCalled() + expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() }) it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { - deps.processValues.mockReturnValue(undefined) + deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) expect(deps.hfValues).toHaveBeenCalledTimes(0) }) @@ -280,6 +283,18 @@ describe('Bootstrapping values', () => { { id: 'user1', initialPassword: 'existing-password' }, { id: 'user2', initialPassword: generatedPassword }, ] + // Pre-processed users (as stored in allSecrets for sealed secret generation) + const processedUsers = usersWithPasswords.map((u: any) => ({ + email: u.email, + firstName: u.firstName, + lastName: u.lastName, + initialPassword: u.initialPassword, + groups: [ + ...(u.isPlatformAdmin ? ['platform-admin'] : []), + ...(u.isTeamAdmin ? ['team-admin'] : []), + ...(u.teams || []).map((t: string) => `team-${t}`), + ], + })) const ca = { a: 'cert' } const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) @@ -304,6 +319,8 @@ describe('Bootstrapping values', () => { addInitialPasswords: jest.fn().mockReturnValue(usersWithPasswords), addPlatformAdmin: jest.fn().mockReturnValue(usersWithPasswords), pathExists: jest.fn().mockReturnValue(true), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), + stripAllSecrets: jest.fn().mockImplementation((v) => v), } }) describe('Creating CA', () => { @@ -331,7 +348,8 @@ describe('Bootstrapping values', () => { deps.getStoredClusterSecrets.mockReturnValue(secrets) deps.generateSecrets.mockReturnValue(allSecrets) await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', allSecrets) + const expected = { ...allSecrets, users: processedUsers } + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { @@ -351,11 +369,10 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) deps.createCustomCA.mockReturnValue(ca) await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith( - 'otomi-generated-passwords', - 'otomi', - mergedSecretsWithGenAndCa, - ) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...mergedSecretsWithGenAndCa, + users: processedUsers, + }) }) it('should not overwrite stored secrets', async () => { deps.loadYaml.mockReturnValue({}) @@ -364,37 +381,47 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) await processValues(deps) expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', generatedSecrets) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...generatedSecrets, + users: processedUsers, + }) }) - it('should only write and return original values', async () => { + it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { deps.loadYaml.mockReturnValue({ cluster: { name: 'bla', provider: 'dida' }, }) - deps.createCustomCA.mockReturnValue({ a: 'cert' }) deps.getStoredClusterSecrets.mockReturnValue({ users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) deps.generateSecrets.mockReturnValue({ gen: 'x' }) deps.createCustomCA.mockReturnValue(ca) const res = await processValues(deps) + // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) + // processedUsers adds groups:[] to each user via element-wise lodash merge expect(deps.writeValues).toHaveBeenNthCalledWith(1, { - cluster: { name: 'bla', provider: 'dida' }, a: 'cert', gen: 'x', + cluster: { name: 'bla', provider: 'dida' }, users: [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: 'generated-password' }, + { id: 'user1', initialPassword: 'existing-password', groups: [] }, + { id: 'user2', initialPassword: 'generated-password', groups: [] }, ], }) - expect(res).toEqual({ + expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) }) - it('should merge original with generated values and write them to env dir', async () => { - const writtenValues = merge( + it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { + // mergedForDisk = merge(originalInput, allSecrets, { users }) + // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { + users: processedUsers, + }) + const expectedDiskValues = merge( + cloneDeep(secrets), cloneDeep(values), - cloneDeep(mergedSecretsWithGenAndCa), + cloneDeep(allSecretsExpected), cloneDeep({ users: usersWithPasswords }), ) deps.loadYaml.mockReturnValue({ ...values, users }) @@ -402,7 +429,25 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) deps.getUsers.mockReturnValue(usersWithPasswords) await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, writtenValues) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + }) + it('should call stripAllSecrets before writing values to disk', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) + await processValues(deps) + expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + }) + it('should still return full allSecrets for bootstrapSealedSecrets', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + const result = await processValues(deps) + // allSecrets should contain full unstripped secrets including pre-processed users + expect(result.allSecrets).toEqual( + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + ) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 96e30c60c7..f5d4c13f60 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -21,7 +21,16 @@ import { secretId, } from 'src/common/k8s' import { getKmsSettings } from 'src/common/repo' -import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils' +import { bootstrapSealedSecrets, stripAllSecrets } from 'src/common/sealed-secrets' +import { + ensureTeamGitOpsDirectories, + getFilename, + getSchemaSecretsPaths, + gucci, + isCore, + loadYaml, + rootDir, +} from 'src/common/utils' import { generateSecrets, writeValues } from 'src/common/values' import { BasicArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' @@ -99,11 +108,6 @@ export const bootstrapSops = async ( await deps.writeFile(targetPath, output) d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - d.info('Copying sops related files') - // add sops related files - const file = '.gitattributes' - await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - // prepare some credential files the first time and crypt some if (!exists) { if (isCli || env.OTOMI_DEV) { @@ -305,8 +309,10 @@ export const processValues = async ( generatePassword, addInitialPasswords, addPlatformAdmin, + getSchemaSecretsPaths, + stripAllSecrets, }, -): Promise> => { +): Promise<{ originalInput: Record; allSecrets: Record }> => { const d = deps.terminal(`cmd:${cmdName}:processValues`) const { VALUES_INPUT } = env d.log(`Loading app values from ${VALUES_INPUT}`) @@ -334,12 +340,32 @@ export const processValues = async ( ) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) - // we have generated all we need, now store everything by merging the original values over all the secrets - await deps.writeValues(merge(cloneDeep(allSecrets), cloneDeep(originalInput), cloneDeep({ users }))) + // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage + const processedUsers = users.map((user: any) => { + const groups: string[] = [] + if (user.isPlatformAdmin) groups.push('platform-admin') + if (user.isTeamAdmin) groups.push('team-admin') + for (const team of user.teams || []) groups.push(`team-${team}`) + return { + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + initialPassword: user.initialPassword, + groups, + } + }) + // Store processed users in allSecrets so they flow into sealed secret generation + allSecrets.users = processedUsers + // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets + // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) + const mergedForDisk = merge(cloneDeep(originalInput), cloneDeep(allSecrets), cloneDeep({ users })) + const secretPaths = await deps.getSchemaSecretsPaths(Object.keys(get(mergedForDisk, 'teamConfig', {}))) + const valuesForDisk = deps.stripAllSecrets(mergedForDisk, secretPaths) + await deps.writeValues(valuesForDisk) // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return originalInput + return { originalInput, allSecrets } } // create file structure based on file entry @@ -434,6 +460,7 @@ export const bootstrap = async ( hfValues, writeValues, bootstrapSops, + bootstrapSealedSecrets, migrate, encrypt, decrypt, @@ -449,10 +476,10 @@ export const bootstrap = async ( } await deps.copyBasicFiles() await deps.migrate() - const originalValues = await deps.processValues() + const { originalInput, allSecrets } = await deps.processValues() await deps.handleFileEntry() - await deps.bootstrapSops() - await ensureTeamGitOpsDirectories(ENV_DIR, originalValues) + await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR, originalInput) + await ensureTeamGitOpsDirectories(ENV_DIR, originalInput) d.log(`Done bootstrapping values`) } @@ -471,7 +498,6 @@ export const module = { handler: async (argv: BasicArguments): Promise => { setParsedArgs(argv) await prepareEnvironment({ skipAllPreChecks: true }) - await decrypt() await bootstrap() await bootstrapGit() }, diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 1605e9a3a8..96f427920e 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -4,11 +4,11 @@ import { prepareEnvironment } from 'src/common/cli' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' +import { getRepo, GitRepoConfig } from 'src/common/git-config' import { waitTillGitRepoAvailable } from 'src/common/gitea' -import { createUpdateConfigMap, createUpdateGenericSecret, k8s } from 'src/common/k8s' +import { hfValues } from 'src/common/hf' +import { createUpdateConfigMap, createUpdateGenericSecret, getK8sSecret, k8s } from 'src/common/k8s' import { getFilename } from 'src/common/utils' -import { getRepo, GitRepoConfig } from 'src/common/git-config' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' import { $, cd } from 'zx' @@ -63,7 +63,7 @@ const commitAndPush = async ( const d = terminal(`cmd:${cmdName}:commitAndPush`) d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' - const { password } = gitConfig ?? getRepo(values) + const { password } = gitConfig ?? (await getRepo(values)) cd(env.ENV_DIR) try { try { @@ -84,8 +84,9 @@ const commitAndPush = async ( } await $`git commit -m ${message} --no-verify`.quiet() } catch (e) { - d.log('commitAndPush error ', e?.message?.replace(password, '****')) - return + const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` + d.error(errorMsg) + throw new Error(errorMsg) } if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1' await retry( @@ -142,11 +143,16 @@ export const commit = async ( await validateValues(overrideArgs) d.info('Preparing values') const values = (await hfValues()) as Record - // Use provided gitConfig if available (operator mode), otherwise read from values (bootstrap/install mode) - const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? getRepo(values) + const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? (await getRepo(values)) if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) + // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during + // the bootstrap phase before install) may have set the URL with unresolved placeholder + // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, + // we need to update the URL with the real credentials. + cd(env.ENV_DIR) + await $`git remote set-url origin ${remote}`.nothrow().quiet() } else { cd(env.ENV_DIR) await setIdentity(username, email) @@ -166,23 +172,35 @@ export async function initialSetupData(): Promise { const values = (await hfValues()) as Record const { domainSuffix } = values.cluster const { hasExternalIDP } = values.otomi - - const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` - const platformAdmin = values.users.find((user: any) => user.email === defaultPlatformAdminEmail) const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' - if (platformAdmin && !hasExternalIDP) { + if (!hasExternalIDP) { + // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) + const usersSecret = await getK8sSecret('users-secrets', 'sealed-secrets') + let platformAdminPassword = '' + if (usersSecret?.usersJson) { + // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string + const users = Array.isArray(usersSecret.usersJson) + ? usersSecret.usersJson + : JSON.parse(String(usersSecret.usersJson)) + const defaultEmail = `platform-admin@${domainSuffix}` + const platformAdmin = users.find((u: any) => u.email === defaultEmail) + platformAdminPassword = platformAdmin?.initialPassword ?? '' + } return { domainSuffix, - username: platformAdmin.email, - password: platformAdmin.initialPassword, + username: `platform-admin@${domainSuffix}`, + password: platformAdminPassword, secretName, } } else { + // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) + const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, - username: values.apps.keycloak.adminUsername, - password: values.apps.keycloak.adminPassword, + username: 'otomi-admin', + password: adminPassword, secretName, } } diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 86d69a4a5a..89967147a5 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -17,6 +17,7 @@ jest.mock('src/common/k8s', () => ({ applyServerSide: jest.fn(), restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), + getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), k8s: { app: jest.fn(), }, @@ -39,9 +40,31 @@ jest.mock('src/common/git-config', () => ({ setGitConfig: jest.fn(), })) -jest.mock('zx', () => ({ - $: jest.fn(), - cd: jest.fn(), +jest.mock('zx', () => { + const mockResult = { exitCode: 0, stdout: '', stderr: '' } + const createMockProcessPromise = () => { + const promise = Promise.resolve(mockResult) + const chainable: any = promise + chainable.nothrow = jest.fn().mockReturnValue(chainable) + chainable.quiet = jest.fn().mockReturnValue(chainable) + return chainable + } + return { + $: jest.fn().mockImplementation(() => createMockProcessPromise()), + cd: jest.fn(), + } +}) + +jest.mock('src/common/sealed-secrets', () => ({ + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), +})) + +jest.mock('src/common/utils', () => ({ + ...jest.requireActual('src/common/utils'), + rootDir: '/test/root', + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), })) jest.mock('./commit', () => ({ @@ -64,11 +87,6 @@ jest.mock('src/common/cli', () => ({ prepareEnvironment: jest.fn(), })) -jest.mock('src/common/utils', () => ({ - ...jest.requireActual('src/common/utils'), - rootDir: '/test/root', -})) - jest.mock('src/common/yargs', () => ({ getParsedArgs: jest.fn().mockReturnValue({}), setParsedArgs: jest.fn(), @@ -111,7 +129,6 @@ describe('Install command', () => { stderr: '', }) mockDeps.deployEssential.mockResolvedValue(true) - mockDeps.$.mockResolvedValue(undefined) }) describe('module configuration', () => { diff --git a/src/cmd/install.ts b/src/cmd/install.ts index b91de56f4e..8a221696cf 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -5,8 +5,20 @@ import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { setGitConfig } from 'src/common/git-config' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS, hfValues } from 'src/common/hf' -import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' -import { getFilename, rootDir } from 'src/common/utils' +import { + applyServerSide, + getDeploymentState, + getHelmReleases, + getK8sSecret, + setDeploymentState, + waitForCRD, +} from 'src/common/k8s' +import { + applySealedSecretManifestsFromDir, + buildSecretToNamespaceMap, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' +import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' import { Argv, CommandModule } from 'yargs' @@ -45,6 +57,65 @@ const retryInstallStep = async ( ) } +/** + * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. + * Derives the list of secrets to wait for from schema x-secret fields via buildSecretToNamespaceMap(). + */ +const waitForSealedSecrets = async ( + timeoutMs = 120000, + intervalMs = 3000, + deps = { getK8sSecret, terminal, buildSecretToNamespaceMap, getSchemaSecretsPaths }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) + + // Build list of secrets to wait for from schema-driven mappings + // We pass empty secrets/teams since we just need the secret names and namespaces + const mappings = await deps.buildSecretToNamespaceMap({}, [], undefined, { + getSchemaSecretsPaths: deps.getSchemaSecretsPaths, + }) + + // Deduplicate by namespace/secretName + const secretsToWait = new Map() + for (const mapping of mappings) { + const key = `${mapping.namespace}/${mapping.secretName}` + if (!secretsToWait.has(key)) { + secretsToWait.set(key, { namespace: mapping.namespace, secretName: mapping.secretName }) + } + } + + if (secretsToWait.size === 0) { + d.info('No sealed secrets to wait for') + return + } + + d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) + const start = Date.now() + + while (Date.now() - start < timeoutMs) { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait.values()) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { + pending.push(`${namespace}/${secretName}`) + } + } + + if (pending.length === 0) { + d.info('All sealed secrets have been decrypted') + return + } + + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + + throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) +} + export const installAll = async () => { const d = terminal(`cmd:${cmdName}:installAll`) const prevState = await getDeploymentState() @@ -65,16 +136,65 @@ export const installAll = async () => { throw new Error('Failed to deploy essential manifests') } + // Deploy sealed-secrets controller right after essentials + d.info('Deploying sealed-secrets controller') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=sealed-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for SealedSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') + + d.info('Applying SealedSecret manifests') + await applySealedSecretManifestsFromDir(env.ENV_DIR) + + d.info('Restarting sealed-secrets controller') + await restartSealedSecretsController() + + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets() + + // Deploy ESO (External Secrets Operator) + d.info('Deploying external-secrets operator') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for ExternalSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'externalsecrets.external-secrets.io') + + d.info('Deploying ESO ClusterSecretStore') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + // Deploy CRDs d.info('Deploying CRDs') await retryInstallStep(applyServerSide, 'charts/kube-prometheus-stack/charts/crds/crds') - // Wait for ServiceMonitor CRD to be established before deploying nginx await retryInstallStep(waitForCRD, 'servicemonitors.monitoring.coreos.com') await retryInstallStep(async () => $`kubectl apply -f charts/tekton-triggers/crds --server-side`) d.info('Deploying charts containing label stage=prep') await hf( { - // 'fileOpts' limits the hf scope and avoids parse errors (we only have basic values at this stage): fileOpts: 'helmfile.d/helmfile-02.init.yaml.gotmpl', labelOpts: ['stage=prep'], logLevel: logLevelString(), @@ -96,18 +216,42 @@ export const installAll = async () => { { streams: { stdout: d.stream.log, stderr: d.stream.error } }, ) + // Deploy cert-manager artifacts (ExternalSecrets, ClusterIssuers, Certificates) + // Must be after app=core (cert-manager CRDs) and after ESO + ClusterSecretStore + d.info('Deploying cert-manager artifacts') + await hf( + { + fileOpts: 'helmfile.d/helmfile-07.init.yaml.gotmpl', + labelOpts: ['name=cert-manager-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + if (!(env.isDev && env.DISABLE_SYNC)) { // Get the git configuration from values const values = (await hfValues()) as Record // Commit to Git repository await commit(true) + const gitBranch = values?.otomi?.git?.branch ?? 'main' await setGitConfig({ repoUrl: values?.otomi?.git?.repoUrl, - branch: values?.otomi?.git?.branch ?? 'main', + branch: gitBranch, email: values?.otomi?.git?.email, }) + // Verify the git push actually succeeded by checking the remote branch exists + d.info('Verifying git push succeeded') + const verifyResult = await $`git -C ${env.ENV_DIR} ls-remote --exit-code --heads origin ${gitBranch}` + .nothrow() + .quiet() + if (verifyResult.exitCode !== 0) { + throw new Error(`Git push verification failed: remote branch ${gitBranch} does not exist after commit`) + } + d.info('Git push verified successfully') + const initialData = await initialSetupData() await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password) await retryInstallStep(createWelcomeConfigMap, initialData.secretName, initialData.domainSuffix) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index bc4e12d6d4..0ad054f1f0 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -21,7 +21,7 @@ import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' import { ARGOCD_APP_PARAMS } from '../common/constants' -import { getSealedSecretsPEM, k8s } from '../common/k8s' +import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' const cmdName = getFilename(__filename) @@ -712,7 +712,12 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - await createCatalogSealedSecret(d, gitea as { adminUsername: string; adminPassword: string }) + const giteaSecrets = await getK8sSecret('gitea-secrets', 'sealed-secrets') + const resolvedGitea = { + adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), + adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), + } + await createCatalogSealedSecret(d, resolvedGitea) secretCreated = true } catch (error) { d.error('Failed to create catalog sealed secret, continuing without it:', error) diff --git a/src/cmd/validate-values.ts b/src/cmd/validate-values.ts index 63d604954b..47eecbe98b 100644 --- a/src/cmd/validate-values.ts +++ b/src/cmd/validate-values.ts @@ -1,5 +1,5 @@ import Ajv, { ValidateFunction } from 'ajv' -import { unset } from 'lodash' +import { cloneDeep, unset } from 'lodash' import { prepareEnvironment } from 'src/common/cli' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -13,6 +13,53 @@ const cmdName = getFilename(__filename) const internalPaths: string[] = ['k8s', 'adminApps', 'teamApps'] +/** + * Remove x-secret properties from `required` arrays throughout the schema. + * Disk values have secrets stripped, so requiring them would always fail validation. + */ +export function removeSecretRequirements(schema: Record): Record { + const cleaned = cloneDeep(schema) + removeSecretRequirementsInPlace(cleaned) + return cleaned +} + +function removeSecretRequirementsInPlace(schema: Record): void { + if (!schema || typeof schema !== 'object') return + + // Collect property names that have x-secret in this node + const secretProps = new Set() + if (schema.properties) { + for (const [propName, propSchema] of Object.entries(schema.properties as Record)) { + if (propSchema && typeof propSchema === 'object' && 'x-secret' in propSchema) { + secretProps.add(propName) + } + } + } + + // Remove secret properties from required array + if (Array.isArray(schema.required) && secretProps.size > 0) { + const filtered = schema.required.filter((r: string) => !secretProps.has(r)) + if (filtered.length === 0) { + // eslint-disable-next-line no-param-reassign + delete schema.required + } else { + // eslint-disable-next-line no-param-reassign + schema.required = filtered + } + } + + // Recurse into all sub-schemas + for (const value of Object.values(schema)) { + if (Array.isArray(value)) { + for (const item of value) { + removeSecretRequirementsInPlace(item) + } + } else if (value && typeof value === 'object') { + removeSecretRequirementsInPlace(value) + } + } +} + // TODO: Accept json path to validate - on empty, validate all export const validateValues = async (argv: HelmArguments = getParsedArgs(), envDir = env.ENV_DIR): Promise => { const d = terminal(`cmd:${cmdName}:validateValues`) @@ -33,12 +80,14 @@ export const validateValues = async (argv: HelmArguments = getParsedArgs(), envD d.info('Loading values-schema.yaml') const valuesSchema = (await loadYaml(`${rootDir}/values-schema.yaml`)) as Record + // Disk values have secrets stripped — remove x-secret fields from required arrays + const adjustedSchema = removeSecretRequirements(valuesSchema) d.debug('Initializing Ajv') const ajv = new Ajv({ allErrors: true, strict: false, strictTypes: false, verbose: true }) d.debug('Compiling Ajv validation') let validate: ValidateFunction try { - validate = ajv.compile(valuesSchema) + validate = ajv.compile(adjustedSchema) } catch (error) { throw new Error(`Schema is invalid: ${chalk.italic(error.message)}`) } diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 7fc12aa385..7f1ce9d1d7 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -1,10 +1,12 @@ import { existsSync } from 'fs' +import { get } from 'lodash' import { decrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' -import { getFilename } from 'src/common/utils' import { getRepo } from 'src/common/git-config' +import { hfValues } from 'src/common/hf' +import { stripAllSecrets } from 'src/common/sealed-secrets' +import { getFilename, getSchemaSecretsPaths } from 'src/common/utils' import { writeValues } from 'src/common/values' import { $, cd } from 'zx' @@ -23,7 +25,7 @@ export const bootstrapGit = async (inValues?: Record): Promise) - const { authenticatedUrl: remote, branch, email, username, password } = getRepo(values) + const { authenticatedUrl: remote, branch, email, username, password } = await getRepo(values) cd(env.ENV_DIR) if (existsSync(`${env.ENV_DIR}/.git`)) { d.info(`Git repo was already bootstrapped, setting identity just in case`) @@ -62,9 +64,12 @@ export const bootstrapGit = async (inValues?: Record): Promise + // Strip ALL secrets before writing to disk — secrets are in SealedSecrets only + const secretPaths = await getSchemaSecretsPaths(Object.keys(get(defaultValues, 'teamConfig', {}))) + const strippedValues = stripAllSecrets(defaultValues, secretPaths) // finally write back the new values without overwriting existing values d.info('Write default values to env repo') - await writeValues(defaultValues) + await writeValues(strippedValues) } if (!existsSync(`${env.ENV_DIR}/.git`)) { diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 6d2e23888f..4883b81d94 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -289,7 +289,7 @@ describe('git-config', () => { }) describe('getRepo', () => { - it('should return full config from values', () => { + it('should return full config from values', async () => { const values = { otomi: { git: { @@ -302,7 +302,7 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result).toEqual({ repoUrl: 'https://github.com/org/repo.git', authenticatedUrl: 'https://admin:s3cret@github.com/org/repo.git', @@ -313,19 +313,19 @@ describe('git-config', () => { }) }) - it('should throw when repoUrl is missing', () => { - expect(() => getRepo({ otomi: { git: {} } })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when repoUrl is missing', async () => { + await expect(getRepo({ otomi: { git: {} } })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when otomi.git is missing', () => { - expect(() => getRepo({ otomi: {} })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when otomi.git is missing', async () => { + await expect(getRepo({ otomi: {} })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when values is empty', () => { - expect(() => getRepo({})).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when values is empty', async () => { + await expect(getRepo({})).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should use GIT_REPO_URL env var in development mode', () => { + it('should use GIT_REPO_URL env var in development mode', async () => { process.env.NODE_ENV = 'development' process.env.GIT_REPO_URL = 'http://localhost:3000/dev/repo.git' @@ -341,9 +341,65 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result.repoUrl).toBe('http://localhost:3000/dev/repo.git') expect(result.authenticatedUrl).toBe('http://admin:s3cret@localhost:3000/dev/repo.git') }) + + it('should fallback to K8s secret when password is a sealed placeholder', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'real-password' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: 'sealed:sealed-secrets/otomi-platform-secrets/git_password', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'sealed-secrets') + expect(result.password).toBe('real-password') + expect(result.authenticatedUrl).toContain('real-password') + }) + + it('should fallback to K8s secret when password is empty', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'from-k8s' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('from-k8s') + }) + + it('should keep empty password when K8s secret also has no password', async () => { + const secretMock = jest.fn().mockResolvedValue(null) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('') + }) }) }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index b92d318b85..3c18e86333 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,6 +1,6 @@ +import type { CoreV1Api } from '@kubernetes/client-node' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' -import type { CoreV1Api } from '@kubernetes/client-node' const d = terminal('common:git-config') @@ -111,7 +111,7 @@ export async function getStoredGitRepoConfig(): Promise, coreV1Api?: C /** * Gets repository configuration from values, constructing the authenticated URL with embedded credentials. + * If password is missing or is an unresolved sealed-secret placeholder, falls back to reading + * the real password from the K8s secret (populated by ESO from SealedSecrets). */ -export const getRepo = (values: Record): GitRepoConfig => { +export const getRepo = async (values: Record, deps = { getK8sSecret }): Promise => { const otomiGit = values?.otomi?.git if (!otomiGit?.repoUrl) { @@ -142,15 +144,29 @@ export const getRepo = (values: Record): GitRepoConfig => { otomiGit.repoUrl = process.env.GIT_REPO_URL } const username = otomiGit?.username - const password = otomiGit?.password + let password = otomiGit?.password ?? '' const email = otomiGit?.email const branch = otomiGit?.branch + // If password is missing or is an unresolved sealed-secret placeholder, + // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) + if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { + try { + const secret = await deps.getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + if (secret?.git_password) { + password = String(secret.git_password) + d.debug('Read git password from K8s secret (ESO)') + } + } catch { + d.warn('Could not read git password from K8s secret, using value from config') + } + } + const repoUrl = otomiGit?.repoUrl as string const url = new URL(repoUrl) url.username = username url.password = password const authenticatedUrl = url.toString() - return { repoUrl, authenticatedUrl, branch, email, username, password } + return { repoUrl, authenticatedUrl, branch, email, username, password } as GitRepoConfig } diff --git a/src/common/repo.ts b/src/common/repo.ts index f37993ca7b..6257c89f60 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -571,7 +571,6 @@ export async function setValuesFile( deps = { pathExists: existsSync, loadValues, writeFile }, ): Promise { const valuesPath = path.join(envDir, 'values-repo.yaml') - // if (await deps.pathExists(valuesPath)) return valuesPath const allValues = await deps.loadValues(envDir) await deps.writeFile(valuesPath, objectToYaml(allValues)) return valuesPath diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts new file mode 100644 index 0000000000..0fa226e9b8 --- /dev/null +++ b/src/common/sealed-secrets.test.ts @@ -0,0 +1,575 @@ +import { pki } from 'node-forge' +import stubs from 'src/test-stubs' +import { + APP_NAMESPACE_MAP, + bootstrapSealedSecrets, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + generateSealedSecretsKeyPair, + getPemFromCertificate, + SECRET_NAME_MAP, + stripAllSecrets, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('zx', () => ({ + $: jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), + }), + }), +})) + +jest.mock('src/common/envalid', () => ({ + env: {}, +})) + +describe('sealed-secrets', () => { + describe('generateSealedSecretsKeyPair', () => { + it('should generate a valid key pair with certificate and private key', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'), + privateKeyToPem: jest + .fn() + .mockReturnValue('-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n'), + } as unknown as typeof pki, + } + + const result = generateSealedSecretsKeyPair(deps) + + expect(deps.pki.rsa.generateKeyPair).toHaveBeenCalledWith(4096) + expect(deps.pki.createCertificate).toHaveBeenCalled() + expect(mockCert.sign).toHaveBeenCalled() + expect(result.certificate).toContain('BEGIN CERTIFICATE') + expect(result.privateKey).toContain('BEGIN RSA PRIVATE KEY') + }) + + it('should set 10-year validity', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: {}, + privateKey: {}, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('cert'), + privateKeyToPem: jest.fn().mockReturnValue('key'), + } as unknown as typeof pki, + } + + generateSealedSecretsKeyPair(deps) + + const notBefore = mockCert.validity.notBefore.getFullYear() + const notAfter = mockCert.validity.notAfter.getFullYear() + expect(notAfter - notBefore).toBe(10) + }) + }) + + describe('getPemFromCertificate', () => { + it('should extract SPKI public key from a certificate', () => { + // Generate a real key pair and certificate for this test + const keys = pki.rsa.generateKeyPair(2048) + const cert = pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 1) + const attrs = [{ name: 'commonName', value: 'test' }] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.sign(keys.privateKey) + const certPem = pki.certificateToPem(cert) + + const result = getPemFromCertificate(certPem) + + expect(result).toContain('BEGIN PUBLIC KEY') + expect(result).toContain('END PUBLIC KEY') + }) + }) + + describe('createSealedSecretsKeySecret', () => { + it('should create secret if it does not exist', async () => { + const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) + const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) + // First call (namespace): success, Second call (check exists): not found (exitCode 1) + // Third call (create): success, Fourth call (label): success + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), + }) // check exists - NOT FOUND + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // create + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // label + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + expect(mock$).toHaveBeenCalledTimes(4) + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + }) + + it('should skip creation if secret already exists', async () => { + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // check exists - FOUND + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + // Should only call namespace and check exists, not create or label + expect(mock$).toHaveBeenCalledTimes(2) + expect(deps.writeFile).not.toHaveBeenCalled() + }) + }) + + describe('buildSecretToNamespaceMap', () => { + it('should group secrets by namespace and secret name with leaf key naming', async () => { + const secrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // All secrets now go to sealed-secrets namespace + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') + }) + + it('should skip kms.sops paths', async () => { + const secrets = { + kms: { sops: { provider: 'age', age: { publicKey: 'pk', privateKey: 'sk' } } }, + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('sealed-secrets') + }) + + it('should serialize users array as single JSON value in users-secrets', async () => { + const secrets = { + users: [ + { + email: 'admin@example.com', + firstName: 'Admin', + lastName: 'User', + initialPassword: 'pass', + groups: ['platform-admin'], + }, + ], + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(2) + const usersMapping = result.find((m) => m.secretName === 'users-secrets') + expect(usersMapping).toBeDefined() + expect(usersMapping!.namespace).toBe('sealed-secrets') + expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) + }) + + it('should handle teamConfig dynamic paths in sealed-secrets namespace', async () => { + const secrets = { + teamConfig: { + 'team-alpha': { someSecret: 'value' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), + } + + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].secretName).toBe('team-team-alpha-settings-secrets') + }) + + it('should filter out mappings with no data', async () => { + const secrets = {} + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(0) + }) + + it('should use leaf key naming for nested secret paths', async () => { + const secrets = { + apps: { + harbor: { core: { secret: 'core-secret-val' } }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.core.secret']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') + }) + + it('should put gitea secrets in sealed-secrets namespace using convention naming', async () => { + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.gitea.adminPassword', + 'apps.gitea.postgresqlPassword', + 'apps.harbor.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // Harbor should use convention naming in sealed-secrets ns + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + + // Gitea should have a gitea-secrets mapping in sealed-secrets ns + const giteaMapping = result.find((m) => m.secretName === 'gitea-secrets') + expect(giteaMapping).toBeDefined() + expect(giteaMapping!.namespace).toBe('sealed-secrets') + expect(giteaMapping!.data).toHaveProperty('adminPassword', 'gitea-pass') + expect(giteaMapping!.data).toHaveProperty('postgresqlPassword', 'pg-pass') + }) + }) + + describe('createSealedSecretManifest', () => { + it('should produce correct SealedSecret structure', async () => { + const mapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'my-password', secretKey: 'my-secret' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), + } + + const result = await createSealedSecretManifest('mock-pem', mapping, deps) + + expect(result.apiVersion).toBe('bitnami.com/v1alpha1') + expect(result.kind).toBe('SealedSecret') + expect(result.metadata.name).toBe('harbor-secrets') + expect(result.metadata.namespace).toBe('sealed-secrets') + expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') + expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') + expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.metadata.name).toBe('harbor-secrets') + expect(result.spec.template.metadata.namespace).toBe('sealed-secrets') + }) + + it('should call encryptSecretItem for each data key', async () => { + const mapping = { + namespace: 'sealed-secrets', + secretName: 'gitea-secrets', + data: { key1: 'val1', key2: 'val2', key3: 'val3' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await createSealedSecretManifest('pem', mapping, deps) + + expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val3') + }) + }) + + describe('writeSealedSecretManifests', () => { + it('should write each manifest to the correct directory', async () => { + const manifests = [ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'sealed-secrets', + }, + spec: { + encryptedData: { key: 'enc' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + type: 'Opaque', + }, + }, + }, + ] + const deps = { + mkdir: jest.fn(), + writeFile: jest.fn(), + objectToYaml: jest.fn().mockReturnValue('yaml-content'), + terminal, + } + + await writeSealedSecretManifests(manifests, '/test', deps) + + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/sealed-secrets', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith( + '/test/env/manifests/ns/sealed-secrets/harbor-secrets.yaml', + 'yaml-content', + ) + }) + }) + + describe('bootstrapSealedSecrets', () => { + it('should generate new key pair when no existing cert found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + const mockManifest = { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'sealed-secrets', + }, + spec: { + encryptedData: { adminPassword: 'encrypted' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + type: 'Opaque', + }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), // No existing cert + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert-pem', + privateKey: 'key-pem', + }), + getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() + expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') + }) + + it('should use existing cert when found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue('existing-cert-pem'), // Existing cert found + generateSealedSecretsKeyPair: jest.fn(), + getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue({}), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).not.toHaveBeenCalled() // Should NOT generate new key + expect(deps.createSealedSecretsKeySecret).not.toHaveBeenCalled() // Should NOT create secret + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should extract team names from secrets', async () => { + const secrets = { + teamConfig: { + alpha: { secret: 'val' }, + beta: { secret: 'val' }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert', + privateKey: 'key', + }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + createSealedSecretManifest: jest.fn(), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) + }) + }) + + describe('APP_NAMESPACE_MAP', () => { + it('should have expected mappings', () => { + expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') + expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') + expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') + expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') + expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') + expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') + expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') + }) + }) + + describe('SECRET_NAME_MAP', () => { + it('should have expected secret name mappings', () => { + expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') + expect(SECRET_NAME_MAP['apps.gitea']).toBe('gitea-secrets') + expect(SECRET_NAME_MAP['apps.keycloak']).toBe('keycloak-secrets') + expect(SECRET_NAME_MAP['otomi']).toBe('otomi-platform-secrets') + expect(SECRET_NAME_MAP['oidc']).toBe('oidc-secrets') + expect(SECRET_NAME_MAP['dns']).toBe('dns-secrets') + }) + }) + + describe('stripAllSecrets', () => { + it('should remove all secret paths from values', () => { + const values = { + apps: { + gitea: { adminPassword: 'secret', postgresqlPassword: 'pg-secret', resources: { cpu: '100m' } }, + }, + oidc: { clientID: 'otomi', clientSecret: 'my-secret', issuer: 'https://example.com' }, + } + const secretPaths = ['apps.gitea.adminPassword', 'apps.gitea.postgresqlPassword', 'oidc.clientSecret'] + + const result = stripAllSecrets(values, secretPaths) + + // Secret values should be removed + expect(result.apps.gitea.adminPassword).toBeUndefined() + expect(result.apps.gitea.postgresqlPassword).toBeUndefined() + expect(result.oidc.clientSecret).toBeUndefined() + // Non-secret values should be preserved + expect(result.apps.gitea.resources).toEqual({ cpu: '100m' }) + expect(result.oidc.clientID).toBe('otomi') + expect(result.oidc.issuer).toBe('https://example.com') + }) + + it('should not modify the original values object', () => { + const values = { + apps: { gitea: { adminPassword: 'secret' } }, + } + const secretPaths = ['apps.gitea.adminPassword'] + + stripAllSecrets(values, secretPaths) + + expect(values.apps.gitea.adminPassword).toBe('secret') + }) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts new file mode 100644 index 0000000000..4c364a4946 --- /dev/null +++ b/src/common/sealed-secrets.ts @@ -0,0 +1,628 @@ +import { encryptSecretItem } from '@linode/kubeseal-encrypt' +import { X509Certificate } from 'crypto' +import { existsSync } from 'fs' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' +import { cloneDeep, get, unset } from 'lodash' +import { pki } from 'node-forge' +import { join } from 'path' +import { terminal } from 'src/common/debug' +import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' +import { objectToYaml } from 'src/common/values' +import { $ } from 'zx' + +const cmdName = 'sealed-secrets' + +/** + * Strip ALL x-secret fields from values before writing to disk. + * Secrets are stored exclusively in SealedSecrets and delivered to apps via ExternalSecrets. + * The values repo contains zero secret values. + */ +export function stripAllSecrets(values: Record, secretPaths: string[]): Record { + const stripped = cloneDeep(values) + for (const secretPath of secretPaths) { + unset(stripped, secretPath) + } + return stripped +} + +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + +export interface SecretMapping { + namespace: string + secretName: string + data: Record +} + +export interface SealedSecretManifest { + apiVersion: string + kind: string + metadata: { + annotations: Record + name: string + namespace: string + } + spec: { + encryptedData: Record + template: { + immutable: boolean + metadata: { name: string; namespace: string } + type: string + } + } +} + +/** + * Mapping from secret path prefix to target Kubernetes namespace. + * Dynamic entries like `teamConfig.{teamId}` are handled separately. + */ +export const APP_NAMESPACE_MAP: Record = { + 'apps.harbor': 'harbor', + 'apps.gitea': 'gitea', + 'apps.keycloak': 'keycloak', + 'apps.grafana': 'grafana', + 'apps.loki': 'monitoring', + 'apps.oauth2-proxy': 'istio-system', + 'apps.oauth2-proxy-redis': 'istio-system', + 'apps.prometheus': 'monitoring', + 'apps.otomi-api': 'otomi', + 'apps.cert-manager': 'cert-manager', + 'apps.kubeflow-pipelines': 'kfp', + otomi: 'otomi', + oidc: 'otomi', + smtp: 'otomi', + dns: 'external-dns', + obj: 'otomi', + license: 'otomi', + users: 'keycloak', + alerts: 'monitoring', + cluster: 'cert-manager', +} + +/** + * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. + * Follows the pattern from createCustomCA() in bootstrap.ts. + */ +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { + const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) + d.info('Generating sealed-secrets RSA key pair') + + const keys = deps.pki.rsa.generateKeyPair(4096) + const cert = deps.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 10) + + const attrs = [ + { name: 'countryName', value: 'NL' }, + { shortName: 'ST', value: 'Utrecht' }, + { name: 'localityName', value: 'Utrecht' }, + { name: 'organizationName', value: 'APL' }, + { shortName: 'OU', value: 'SealedSecrets' }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { + name: 'basicConstraints', + cA: true, + }, + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + keyEncipherment: true, + }, + ]) + cert.sign(keys.privateKey) + + const certificate = deps.pki.certificateToPem(cert).replaceAll('\r\n', '\n') + const privateKey = deps.pki.privateKeyToPem(keys.privateKey).replaceAll('\r\n', '\n') + + d.info('Generated sealed-secrets key pair') + return { certificate, privateKey } +} + +/** + * Extract SPKI PEM public key from a PEM-encoded X.509 certificate. + * Uses Node.js crypto.X509Certificate (same approach as getSealedSecretsPEM() in k8s.ts). + */ +export const getPemFromCertificate = (certificate: string): string => { + const x509 = new X509Certificate(certificate) + const exported = x509.publicKey.export({ format: 'pem', type: 'spki' }) + return typeof exported === 'string' ? exported : exported.toString('utf-8') +} + +/** + * Get the existing sealed-secrets certificate from the cluster if it exists. + * Returns the certificate PEM string or undefined if not found. + */ +export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) + + const result = + await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` + .nothrow() + .quiet() + + if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { + d.info('No existing sealed-secrets-key found') + return undefined + } + + try { + const certBase64 = result.stdout.replace(/'/g, '') + const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + d.info('Found existing sealed-secrets-key certificate') + return cert + } catch { + d.warn('Failed to decode existing certificate') + return undefined + } +} + +/** + * Create the sealed-secrets namespace and TLS secret in Kubernetes. + * The controller will pick up this pre-created key on startup. + * IMPORTANT: This only creates the secret if it doesn't already exist. + */ +export const createSealedSecretsKeySecret = async ( + certificate: string, + privateKey: string, + deps = { $, terminal, writeFile, mkdir }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) + + // Create namespace if it doesn't exist + await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) + + // Check if secret already exists + const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() + if (existingSecret.exitCode === 0) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } + + d.info('Creating sealed-secrets TLS secret') + + // Write temp files for kubectl create secret tls + const tmpDir = '/tmp/sealed-secrets-bootstrap' + await deps.mkdir(tmpDir, { recursive: true }) + const certPath = `${tmpDir}/tls.crt` + const keyPath = `${tmpDir}/tls.key` + await deps.writeFile(certPath, certificate) + await deps.writeFile(keyPath, privateKey) + + // Create the TLS secret (only if it doesn't exist) + const result = + await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` + .nothrow() + .quiet() + if (result.exitCode !== 0) { + d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) + return + } + + // Label the secret so the controller picks it up + const labelResult = + await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` + .nothrow() + .quiet() + if (labelResult.stderr) d.error(labelResult.stderr) + + d.info('Created sealed-secrets TLS secret with key label') +} + +/** + * Resolve the namespace for a given secret path. + * All core secrets go to 'sealed-secrets' namespace for ESO access. + * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. + */ +const resolveNamespace = (secretPath: string): string | undefined => { + // Check for teamConfig dynamic paths + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return 'sealed-secrets' + } + + // Check if this path matches any known prefix + const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return 'sealed-secrets' + } + } + + return undefined +} + +// Map specific path prefixes to secret names +export const SECRET_NAME_MAP: Record = { + 'apps.harbor': 'harbor-secrets', + 'apps.gitea': 'gitea-secrets', + 'apps.keycloak': 'keycloak-secrets', + 'apps.grafana': 'grafana-secrets', + 'apps.loki': 'loki-secrets', + 'apps.oauth2-proxy': 'oauth2-proxy-secrets', + 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', + 'apps.prometheus': 'prometheus-secrets', + 'apps.otomi-api': 'otomi-api-secrets', + 'apps.cert-manager': 'cert-manager-secrets', + 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', + otomi: 'otomi-platform-secrets', + oidc: 'oidc-secrets', + smtp: 'smtp-secrets', + dns: 'dns-secrets', + obj: 'obj-storage-secrets', + license: 'license-secrets', + users: 'users-secrets', + alerts: 'alerts-secrets', + cluster: 'cluster-secrets', +} + +/** + * Find the group prefix for a secret path. + * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + */ +const findGroupPrefix = (secretPath: string): string | undefined => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `teamConfig.${teamMatch[1]}` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return prefix + } + } + + // Fallback: use first two path segments + const parts = secretPath.split('.') + if (parts.length >= 2) { + return parts.slice(0, 2).join('.') + } + return undefined +} + +/** + * Derive a K8s secret name from the secret path prefix. + */ +const deriveSecretName = (secretPath: string): string => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `team-${teamMatch[1]}-settings-secrets` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return SECRET_NAME_MAP[prefix] + } + } + + // Fallback: derive from first two path segments + const parts = secretPath.split('.') + return `${parts.slice(0, 2).join('-')}-secrets` +} + +/** + * Build a mapping from secrets to their target namespaces and K8s secret names. + * Groups secret paths by namespace and secret name. + */ +export const buildSecretToNamespaceMap = async ( + secrets: Record, + teams: string[], + allValues?: Record, + deps = { getSchemaSecretsPaths }, +): Promise => { + const secretPaths = await deps.getSchemaSecretsPaths(teams) + const flat = flattenObject(secrets) + const allFlat = allValues ? flattenObject(allValues) : flat + + // Group by namespace + secretName + const groupMap = new Map() + + for (const secretPath of secretPaths) { + // Skip SOPS-related paths + if (secretPath.startsWith('kms.sops')) continue + // Handle 'users' path specially — serialize pre-processed users array as single JSON value + if (secretPath === 'users') { + const usersData = secrets.users + if (Array.isArray(usersData) && usersData.length > 0) { + const namespace = 'sealed-secrets' + const secretName = 'users-secrets' + const groupKey = `${namespace}/${secretName}` + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + const mapping = groupMap.get(groupKey)! + mapping.data.usersJson = JSON.stringify(usersData) + } + continue + } + + const namespace = resolveNamespace(secretPath) + if (!namespace) continue + + const secretName = deriveSecretName(secretPath) + const groupKey = `${namespace}/${secretName}` + + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + + const mapping = groupMap.get(groupKey)! + + // Find the group prefix (e.g., 'apps.harbor' for 'apps.harbor.adminPassword') + const groupPrefix = findGroupPrefix(secretPath) + + // Find all flat keys that match this secret path + for (const [flatKey, value] of Object.entries(flat)) { + if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { + // Use leaf key: strip the group prefix to get relative path + const relativePath = + groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) + ? flatKey.slice(groupPrefix.length + 1) + : flatKey + const dataKey = relativePath.replace(/\./g, '_') + if (value !== undefined && value !== null && value !== '') { + mapping.data[dataKey] = String(value) + } + } + } + } + + // Filter out empty mappings + return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) +} + +/** + * Create a SealedSecret manifest by encrypting each data value. + * Follows the pattern from createCatalogSealedSecret() in migrate.ts. + */ +export const createSealedSecretManifest = async ( + pem: string, + mapping: SecretMapping, + deps = { encryptSecretItem }, +): Promise => { + const encryptedData: Record = {} + for (const [key, value] of Object.entries(mapping.data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, mapping.namespace, value) + } + + return { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: mapping.secretName, + namespace: mapping.namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: mapping.secretName, namespace: mapping.namespace }, + type: 'Opaque', + }, + }, + } +} + +/** + * Write SealedSecret manifests to the env/manifests/ns directory. + */ +export const writeSealedSecretManifests = async ( + manifests: SealedSecretManifest[], + envDir: string, + deps = { mkdir, writeFile, objectToYaml, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) + + for (const manifest of manifests) { + // /env/manifests/ns/argocd/ + const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` + await deps.mkdir(dir, { recursive: true }) + const filePath = `${dir}/${manifest.metadata.name}.yaml` + d.info(`Writing sealed secret to ${filePath}`) + await deps.writeFile(filePath, deps.objectToYaml(manifest)) + } +} + +/** + * Apply SealedSecret manifests to the Kubernetes cluster. + * Creates namespaces if needed and applies the SealedSecret resources. + */ +export const applySealedSecretManifests = async ( + manifests: SealedSecretManifest[], + deps = { $, terminal, objectToYaml }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) + + // Group manifests by namespace + const byNamespace = new Map() + for (const manifest of manifests) { + const ns = manifest.metadata.namespace + if (!byNamespace.has(ns)) { + byNamespace.set(ns, []) + } + byNamespace.get(ns)!.push(manifest) + } + + // Ensure namespaces exist and apply manifests + for (const [namespace, nsManifests] of byNamespace) { + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + for (const manifest of nsManifests) { + d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) + const yaml = deps.objectToYaml(manifest) + const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + } + } + } + + d.info(`Applied ${manifests.length} SealedSecret manifests to cluster`) +} + +/** + * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * This should be called during install, after the sealed-secrets controller is deployed. + */ +export const applySealedSecretManifestsFromDir = async ( + envDir: string, + deps = { $, terminal, readdir, readFile, existsSync }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) + const manifestsDir = join(envDir, 'env/manifests/ns') + + if (!deps.existsSync(manifestsDir)) { + d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) + return + } + + d.info(`Applying SealedSecret manifests from ${manifestsDir}`) + + // Read all namespace directories + const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) + let appliedCount = 0 + + for (const nsEntry of namespaces) { + if (!nsEntry.isDirectory()) continue + const namespace = nsEntry.name + const nsDir = join(manifestsDir, namespace) + + // Ensure namespace exists with proper labels + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + // Read all YAML files in the namespace directory + const files = await deps.readdir(nsDir) + for (const file of files) { + if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue + const filePath = join(nsDir, file) + d.info(`Applying SealedSecret from ${filePath}`) + + const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) + } else { + appliedCount += 1 + } + } + } + + d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) +} + +/** + * Restart the sealed-secrets controller to ensure it uses the correct key. + * This is needed because if the controller starts before the sealed-secrets-key secret exists, + * it will generate its own key. Restarting forces it to pick up the existing key. + */ +export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) + d.info('Restarting sealed-secrets controller to ensure correct key is used') + + const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() + if (result.exitCode !== 0) { + d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + return + } + + d.info('Waiting for sealed-secrets controller rollout') + const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` + .nothrow() + .quiet() + if (waitResult.exitCode !== 0) { + d.warn(`Rollout status check failed: ${waitResult.stderr}`) + } else { + d.info('Sealed-secrets controller restarted successfully') + } +} + +/** + * Orchestrator: bootstrap sealed secrets for the platform. + * Replaces bootstrapSops(). + */ +export const bootstrapSealedSecrets = async ( + secrets: Record, + envDir: string, + allValues?: Record, + deps = { + terminal, + generateSealedSecretsKeyPair, + getPemFromCertificate, + createSealedSecretsKeySecret, + getExistingSealedSecretsCert, + buildSecretToNamespaceMap, + createSealedSecretManifest, + writeSealedSecretManifests, + encryptSecretItem, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) + d.info('Bootstrapping sealed secrets') + + // 1. Check if there's an existing sealed-secrets key in the cluster + const existingCert = await deps.getExistingSealedSecretsCert() + + let pem: string + if (existingCert) { + // Use existing certificate for encryption + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + // Generate new key pair and create the secret + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } + + // 5. Build secret-to-namespace mapping + const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) + + // 6. Create SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + + // 7. Write SealedSecret manifests to disk + // Note: These manifests are applied later during install, after the sealed-secrets + // controller is deployed and the SealedSecret CRD is available. + await deps.writeSealedSecretManifests(manifests, envDir) + + d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) +} diff --git a/src/common/values.ts b/src/common/values.ts index 157ee2fc2c..de44be2dbf 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, omit, pick, set } from 'lodash' +import { cloneDeep, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -9,18 +9,8 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' -import { - extract, - flattenObject, - getSchemaSecretsPaths, - getValuesSchema, - gucci, - loadYaml, - pkg, - removeBlankAttributes, -} from './utils' - import { saveValues } from './repo' +import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -114,22 +104,13 @@ export const writeValuesToFile = async ( /** * Writes new values to the repo. Will keep the original values if `overwrite` is `false`. + * Secret values are written as-is — they are protected by SealedSecrets on the cluster side, + * and child secrets are derived via ESO ExternalSecret CRs. */ export const writeValues = async (inValues: Record, overwrite = false): Promise => { const d = terminal('common:values:writeValues') d.debug('Writing values: ', inValues) - hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = inValues - const teams = Object.keys(get(inValues, 'teamConfig', {})) - const cleanSecretPaths = await getSchemaSecretsPaths(teams) - d.debug('cleanSecretPaths: ', cleanSecretPaths) - // separate out the secrets - const secrets = removeBlankAttributes(pick(values, cleanSecretPaths)) - d.debug('secrets: ', JSON.stringify(secrets, null, 2)) - // from the plain values - const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, plainValues, secrets) - + await saveValues(env.ENV_DIR, inValues, {}) d.info('All values were written to ENV_DIR') } diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 55809ea5d5..02cb411730 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -4,6 +4,31 @@ import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' +const mockZx = jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }), + }), +}) + +jest.mock('zx', () => ({ + $: (...args: any[]) => mockZx(...args), +})) + +jest.mock('../common/envalid', () => ({ + env: { + GIT_PROTOCOL: 'http', + GIT_URL: 'gitea-http.gitea.svc.cluster.local', + GIT_PORT: '3000', + }, +})) + +jest.mock('./validators', () => ({ + operatorEnv: { + GIT_ORG: 'otomi', + GIT_REPO: 'values', + }, +})) + jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -236,10 +261,18 @@ describe('Installer', () => { }) describe('isInstalled', () => { - test('should return completed status when ConfigMap exists', async () => { + test('should return completed status when ConfigMap exists and git repo has main branch', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) + // getK8sSecret returns credentials for git verification + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote succeeds (main branch exists) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const isInstalled = await installer.isInstalled() @@ -248,6 +281,23 @@ describe('Installer', () => { expect(mockAplOps.install).not.toHaveBeenCalled() }) + test('should return false when status is completed but git repo has no main branch', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote fails (main branch does not exist) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 2 }), + }), + }) + + const isInstalled = await installer.isInstalled() + + expect(isInstalled).toBe(false) + }) + test('should return true when ConfigMap does not exist', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue(null) @@ -280,6 +330,19 @@ describe('Installer', () => { expect(k8s.getK8sConfigMap).toHaveBeenCalledWith('apl-operator', 'apl-installation-status', mockCoreApi) expect(isInstalled).toBe(false) }) + + test('should return true when git verification fails (gitea not ready)', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + // getK8sSecret throws (cluster issues) + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + + const isInstalled = await installer.isInstalled() + + // Should assume installed when verification can't be performed + expect(isInstalled).toBe(true) + }) }) describe('setEnvAndCreateSecrets', () => { @@ -292,10 +355,13 @@ describe('Installer', () => { expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') }) - test('should handle failure when SOPS key not found in secret', async () => { + test('should skip gracefully when SOPS key not found in secret (SealedSecrets in use)', async () => { ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(null) - await expect(installer.setEnvAndCreateSecrets()).rejects.toThrow('SOPS_AGE_KEY not found in secret') + await installer.setEnvAndCreateSecrets() + + // Should not throw — SOPS is no longer required (replaced by SealedSecrets + ESO) + expect(process.env.SOPS_AGE_KEY).toBe('') }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 72cb74b749..5bd01f228b 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,4 +1,5 @@ import * as process from 'node:process' +import { $ } from 'zx' import { terminal } from '../common/debug' import { getGitConfigData, @@ -11,6 +12,7 @@ import { hfValues } from '../common/hf' import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' export class Installer { private d = terminal('operator:installer') @@ -28,7 +30,34 @@ export class Installer { await this.updateInstallationStatus('completed', -1) return true } - return installStatus === 'completed' + if (installStatus === 'completed') { + // Verify the git repo actually has content - the previous install may have + // marked status as completed but the pod was killed before the git push finished + const gitRepoHasContent = await this.verifyGitRepoHasMainBranch() + if (!gitRepoHasContent) { + this.d.warn('Installation marked as completed but git repo has no main branch - will re-install') + return false + } + return true + } + return false + } + + private async verifyGitRepoHasMainBranch(): Promise { + try { + // Get credentials from K8s secret (created by Helm at deploy time) + const creds = await getK8sSecret('gitea-credentials', 'apl-operator') + const username = creds?.GIT_USERNAME ?? 'otomi-admin' + const password = creds?.GIT_PASSWORD ?? '' + const repoUrl = `${process.env.GIT_PROTOCOL}://${username}:${password}@${process.env.GIT_URL}:${process.env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` + const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + return result.exitCode === 0 + } catch { + // If we can't check (e.g. gitea not ready yet), assume it's fine + // The operator will detect the issue later during git polling + this.d.warn('Could not verify git repo - gitea may not be ready yet') + return true + } } public async initialize() { @@ -102,10 +131,14 @@ export class Installer { private async setupSopsEnvironment() { const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - if (!aplSopsSecret?.SOPS_AGE_KEY) { - throw new Error('SOPS_AGE_KEY not found in secret') + if (aplSopsSecret?.SOPS_AGE_KEY) { + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.debug('Using existing sops credentials from secret') + } else { + // SOPS is no longer used (replaced by SealedSecrets + ESO). + // Skip hfValues() call which requires the git repo that may not exist yet. + this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') } - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY } // public for testing. This method should only be used if you are certain there are values locally. diff --git a/values-schema.yaml b/values-schema.yaml index 770310ae14..6efea064ae 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -903,7 +903,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - x-secret: 'admin' + default: 'admin' securityContext: additionalProperties: uniqueItems: true @@ -1644,7 +1644,6 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: - x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: @@ -1704,6 +1703,12 @@ properties: $ref: '#/definitions/rawValues' enabled: type: boolean + adminUsername: + type: string + default: otomi-admin + adminPassword: + type: string + x-secret: '{{ randAlphaNum 20 }}' postgresqlPassword: type: string description: This password was generated and cannot be changed without manual intervention. @@ -1791,7 +1796,6 @@ properties: type: string x-secret: '{{ randAlphaNum 32 }}' required: - - secret - credentials databaseMaxConnections: type: number @@ -1899,6 +1903,7 @@ properties: x-secret: '' adminUsername: type: string + default: otomi-admin theme: type: string default: otomi @@ -2729,6 +2734,7 @@ properties: $ref: '#/definitions/url' clientID: $ref: '#/definitions/wordCharacterPattern' + x-secret: '' clientSecret: type: string x-secret: '' @@ -2748,7 +2754,6 @@ properties: default: sub required: - clientID - - clientSecret - issuer otomi: additionalProperties: false diff --git a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl index 36fa0f144a..a89537c3ca 100644 --- a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl +++ b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl @@ -19,13 +19,32 @@ resources: hasArgocd: "{{ $v.apps.argocd.enabled | toString }}" domainSuffix: "{{ $v.cluster.domainSuffix }}" teamConfig: {{ $teamConfig | toJson }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-gitea-operator-secret namespace: apl-gitea-operator - data: - giteaPassword: {{ $v.otomi.git.password | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-gitea-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + giteaPassword: '{{ "{{ .gitPassword | toString }}" }}' + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 1a58d1fec9..72ad5eb791 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -11,17 +11,36 @@ {{- $teamNamespaces := $teamNamespaces | sortAlpha | toJson }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-harbor-operator-secret namespace: apl-harbor-operator - data: - harborPassword: {{ $h.adminPassword | b64enc }} - harborUser: {{ "admin" | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-harbor-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + harborPassword: '{{ "{{ .harborAdminPassword | toString }}" }}' + harborUser: admin + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + data: + - secretKey: harborAdminPassword + remoteRef: + key: harbor-secrets + property: adminPassword + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 5a3a8609c4..2516d98edf 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -21,31 +21,54 @@ {{- $k := $c | get "keycloak" }} {{- $doms := tpl (readFile "../../helmfile.d/snippets/domains.gotmpl") $v | fromYaml }} {{- $joinTpl := readFile "../../helmfile.d/utils/joinListWithSep.gotmpl" }} -{{ $users := list }} -{{- range $user := $v.users }} - {{ $groups := list }} - {{- if $user.isPlatformAdmin }}{{ $groups = append $groups "platform-admin" }}{{ end }} - {{- if $user.isTeamAdmin }}{{ $groups = append $groups "team-admin" }}{{ end }} - {{- range $team := $user | get "teams" list }}{{ $groups = append $groups (print "team-" $team) }}{{ end }} - {{- $users = append $users (dict "email" $user.email "firstName" $user.firstName "lastName" $user.lastName "initialPassword" $user.initialPassword "groups" $groups) }} -{{- end }} -{{- $users := $users | toJson }} - resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-keycloak-operator-secret namespace: apl-keycloak-operator - data: - KEYCLOAK_ADMIN: {{ .Values.apps.keycloak.adminUsername | b64enc }} - KEYCLOAK_ADMIN_PASSWORD: {{ $k.adminPassword | b64enc }} - KEYCLOAK_CLIENT_SECRET: {{ $k.idp.clientSecret | b64enc }} - USERS: {{ $users | b64enc }} - {{- if $v.otomi.hasExternalIDP }} - IDP_CLIENT_ID: {{ $oi.clientID | b64enc}} - IDP_CLIENT_SECRET: {{ $oi.clientSecret | b64enc }} - {{- end }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-keycloak-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + KEYCLOAK_ADMIN: {{ $k.adminUsername }} + KEYCLOAK_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + KEYCLOAK_CLIENT_SECRET: '{{ "{{ .idpClientSecret | toString }}" }}' + USERS: '{{ "{{ .usersJson | toString }}" }}' + {{- if $v.otomi.hasExternalIDP }} + IDP_CLIENT_ID: '{{ "{{ .oidcClientID | toString }}" }}' + IDP_CLIENT_SECRET: '{{ "{{ .oidcClientSecret | toString }}" }}' + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword + - secretKey: idpClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - secretKey: usersJson + remoteRef: + key: users-secrets + property: usersJson + {{- if $v.otomi.hasExternalIDP }} + - secretKey: oidcClientID + remoteRef: + key: oidc-secrets + property: clientID + - secretKey: oidcClientSecret + remoteRef: + key: oidc-secrets + property: clientSecret + {{- end }} - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl new file mode 100644 index 0000000000..1ad5c4a39b --- /dev/null +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -0,0 +1,25 @@ +{{- $v := .Values }} + +resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-credentials + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-credentials + creationPolicy: Owner + template: + type: Opaque + data: + GIT_USERNAME: {{ $v.otomi.git.username | default "otomi-admin" }} + GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-platform-secrets + property: git_password diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index d3bc151b2a..03463b273a 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -2,7 +2,6 @@ {{- $o := $v.apps | get "apl-operator" }} {{- $version := $v.otomi.version }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} -{{- $g := $v.apps.gitea }} {{- $kms := $v | get "kms" dict }} image: {{- if $v.otomi.linodeLkeImageRepository }} @@ -19,11 +18,3 @@ imagePullSecrets: resources: {{- toYaml $o.resources.operator | nindent 2 }} kms: {{- $kms | toYaml | nindent 2 }} - -git: - password: {{ $v.otomi.git.password | quote }} - username: {{ $v.otomi.git.username | quote }} - email: {{ $v.otomi.git.email | quote }} - repoUrl: {{ $v.otomi.git.repoUrl | quote }} - branch: {{ $v.otomi.git.branch | quote }} - diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 8fdc599015..478058fdc1 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -10,41 +10,89 @@ resources: custom-ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} {{- if contains "gitea-http.gitea.svc.cluster.local" $v.otomi.git.repoUrl }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ printf "https://%s" $v._derived.giteaDomain | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} - - apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ printf "https://%s" $v._derived.giteaDomain }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea-internal namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | b64enc | quote }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea-internal + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | quote }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password {{- else }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-git namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ $v.otomi.git.repoUrl | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-git + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ $v.otomi.git.repoUrl }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password {{- end }} diff --git a/values/argocd/argocd.gotmpl b/values/argocd/argocd.gotmpl index 6017fee452..c356f32639 100644 --- a/values/argocd/argocd.gotmpl +++ b/values/argocd/argocd.gotmpl @@ -205,8 +205,7 @@ configs: {{- end }} secret: - extra: - oidc.clientSecret: {{ $k.idp.clientSecret | quote }} + extra: {} params: server.insecure: true # nginx terminates tls # -- Number of application status processors diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index 6f7b113888..e7f0df883f 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -6,31 +6,96 @@ resources: {{- if and $v.otomi.hasExternalDNS (or (not (hasKey $p "aws")) ($p | get "aws.credentials.secretKey" nil)) }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - name: "external-dns" - data: - {{- if hasKey $p "google" }} - secret: "{{ $p.google.serviceAccountKey | b64enc }}" - {{- else if hasKey $p "akamai" }} - access_token: {{ $p.akamai.accessToken | b64enc | quote }} - client_token: {{ $p.akamai.clientToken | b64enc | quote }} - client_secret: {{ $p.akamai.clientSecret | b64enc | quote }} - {{- else if hasKey $p "azure-private-dns" }} - secret: "{{ $p | get "azure-private-dns.aadClientSecret" | b64enc }}" - {{- else if hasKey $p "azure" }} - secret: "{{ $p.azure.aadClientSecret | b64enc }}" - {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} - secret: "{{ $p.aws.credentials.secretKey | b64enc }}" - {{- else if hasKey $p "digitalocean" }} - secret: "{{ $p.digitalocean.apiToken | b64enc }}" - {{- else if hasKey $p "cloudflare" }} - secret: "{{ $p.cloudflare.apiToken | b64enc }}" - {{- else if hasKey $p "linode" }} - secret: "{{ $p.linode.apiToken | b64enc }}" - {{- end }} + name: external-dns + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: external-dns + creationPolicy: Owner + template: + type: Opaque + data: + {{- if hasKey $p "google" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "akamai" }} + access_token: '{{ "{{ .access_token | toString }}" }}' + client_token: '{{ "{{ .client_token | toString }}" }}' + client_secret: '{{ "{{ .client_secret | toString }}" }}' + {{- else if hasKey $p "azure-private-dns" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "azure" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "digitalocean" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "cloudflare" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "linode" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- end }} + data: + - secretKey: secret + remoteRef: + key: dns-secrets + {{- if hasKey $p "google" }} + property: provider_google_serviceAccountKey + {{- else if hasKey $p "akamai" }} + property: provider_akamai_clientSecret + - secretKey: access_token + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken + - secretKey: client_token + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: client_secret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + {{- else if hasKey $p "azure-private-dns" }} + property: provider_azure-private-dns_aadClientSecret + {{- else if hasKey $p "azure" }} + property: provider_azure_aadClientSecret + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + property: provider_aws_credentials_secretKey + {{- else if hasKey $p "digitalocean" }} + property: provider_digitalocean_apiToken + {{- else if hasKey $p "cloudflare" }} + property: provider_cloudflare_apiToken + {{- else if hasKey $p "linode" }} + property: provider_linode_apiToken + {{- end }} {{- end }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: custom-ca + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: custom-ca + creationPolicy: Owner + template: + type: Opaque + data: + tls.crt: {{ $cm.customRootCA | quote }} + tls.key: '{{ "{{ .customRootCAKey | toString }}" }}' + data: + - secretKey: customRootCAKey + remoteRef: + key: cert-manager-secrets + property: customRootCAKey - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: @@ -147,15 +212,29 @@ resources: {{- end }} {{- end }} {{- if eq $cm.issuer "byo-wildcard-cert" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-byo-wildcard-cert namespace: istio-system - type: kubernetes.io/tls - data: - tls.crt: {{ $cm.byoWildcardCert | b64enc }} - tls.key: {{ $cm.byoWildcardCertKey | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-byo-wildcard-cert + creationPolicy: Owner + template: + type: kubernetes.io/tls + data: + tls.crt: {{ $cm.byoWildcardCert | quote }} + tls.key: '{{ "{{ .byoWildcardCertKey | toString }}" }}' + data: + - secretKey: byoWildcardCertKey + remoteRef: + key: cert-manager-secrets + property: byoWildcardCertKey {{- end }} {{- if or (eq $cm.issuer "letsencrypt" ) (eq $cm.issuer "custom-ca" ) }} - apiVersion: cert-manager.io/v1 @@ -179,4 +258,4 @@ resources: - key encipherment - ocsp signing {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/values/external-dns/external-dns-raw.gotmpl b/values/external-dns/external-dns-raw.gotmpl index e70774372c..9f699a9c1a 100644 --- a/values/external-dns/external-dns-raw.gotmpl +++ b/values/external-dns/external-dns-raw.gotmpl @@ -3,103 +3,252 @@ {{- with $d.provider | get "akamai" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: akamai-dns - data: - EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: "{{.clientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: akamai-dns + creationPolicy: Owner + template: + type: Opaque + data: + EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: '{{ "{{ .clientSecret | toString }}" }}' + EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN: '{{ "{{ .clientToken | toString }}" }}' + EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN: '{{ "{{ .accessToken | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + - secretKey: clientToken + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: accessToken + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken {{- end }} {{- with $d.provider | get "linode" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-dns-api-token - data: - LINODE_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-dns-api-token + creationPolicy: Owner + template: + type: Opaque + data: + LINODE_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_linode_apiToken {{- end }} {{- with $d.provider | get "digitalocean" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: do-token - data: - DO_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: do-token + creationPolicy: Owner + template: + type: Opaque + data: + DO_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_digitalocean_apiToken {{- end }} {{- with $d.provider | get "cloudflare" nil }} {{- with .apiToken }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_TOKEN: "{{ . | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiToken {{- end }} {{- with .apiSecret }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_KEY: "{{ . | b64enc }}" - CF_API_EMAIL: "{{ $d.provider.cloudflare.email | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_KEY: '{{ "{{ .apiSecret | toString }}" }}' + CF_API_EMAIL: {{ $d.provider.cloudflare.email | quote }} + data: + - secretKey: apiSecret + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiSecret {{- end }} {{- end }} {{- with $d.provider | get "aws" nil }} {{- with .credentials }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: aws-dns-credentials - data: - AWS_ACCESS_KEY_ID: "{{ .accessKey | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ .secretKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: aws-dns-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: '{{ "{{ .accessKey | toString }}" }}' + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: accessKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_accessKey + - secretKey: secretKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_secretKey {{- end }} {{- end }} {{- with $d.provider | get "azure" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure_aadClientSecret {{- end }} {{- with $d.provider | get "azure-private-dns" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-private-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-private-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure-private-dns_aadClientSecret {{- end }} {{- with $d.provider | get "google" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: google-dns - data: - GOOGLE_APPLICATION_CREDENTIALS: "{{ .serviceAccountKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: google-dns + creationPolicy: Owner + template: + type: Opaque + data: + GOOGLE_APPLICATION_CREDENTIALS: '{{ "{{ .serviceAccountKey | toString }}" }}' + data: + - secretKey: serviceAccountKey + remoteRef: + key: dns-secrets + property: provider_google_serviceAccountKey {{- end }} diff --git a/values/external-dns/external-dns.gotmpl b/values/external-dns/external-dns.gotmpl index a3737c9c57..e5f1d5e83c 100644 --- a/values/external-dns/external-dns.gotmpl +++ b/values/external-dns/external-dns.gotmpl @@ -50,9 +50,15 @@ env: - name: EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN value: "{{ $dns.provider.akamai.host }}" - name: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - value: "{{ $dns.provider.akamai.clientToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - name: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - value: "{{ $dns.provider.akamai.accessToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - name: EXTERNAL_DNS_AKAMAI_CLIENT_SECRET valueFrom: secretKeyRef: diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl new file mode 100644 index 0000000000..c543f54b1e --- /dev/null +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -0,0 +1,47 @@ +{{- $v := .Values }} + +resources: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: eso-store-sa + namespace: external-secrets + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: eso-core-secret-reader + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: eso-core-secret-reader-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eso-core-secret-reader + subjects: + - kind: ServiceAccount + name: eso-store-sa + namespace: external-secrets + - apiVersion: external-secrets.io/v1beta1 + kind: ClusterSecretStore + metadata: + name: core-secrets-store + spec: + provider: + kubernetes: + remoteNamespace: sealed-secrets + server: + url: "https://kubernetes.default.svc" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + namespace: external-secrets + key: ca.crt + auth: + serviceAccount: + name: eso-store-sa + namespace: external-secrets diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl new file mode 100644 index 0000000000..30ba6ada8c --- /dev/null +++ b/values/external-secrets/external-secrets.gotmpl @@ -0,0 +1,9 @@ +{{- $v := .Values }} +{{- $app := $v.apps | get "sealed-secrets" }} + +replicaCount: 1 + +resources: {{- $app.resources.operator | toYaml | nindent 2 }} + +image: + repository: ghcr.io/external-secrets/external-secrets diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 792c355e5f..067dfb8644 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -1,12 +1,25 @@ {{- $v := .Values }} -{{- $g := $v.apps.gitea }} resources: -- apiVersion: v1 - kind: Secret - type: kubernetes.io/basic-auth +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: gitea-db-secret - data: - username: "{{ "gitea" | b64enc }}" - password: "{{ $g.postgresqlPassword | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-db-secret + creationPolicy: Owner + template: + type: kubernetes.io/basic-auth + data: + username: gitea + password: '{{ "{{ .postgresqlPassword | toString }}" }}' + data: + - secretKey: postgresqlPassword + remoteRef: + key: gitea-secrets + property: postgresqlPassword diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 02b2cc15e8..2098889f8d 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -13,6 +13,28 @@ {{- end }} resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-admin-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $v.otomi.git.username | default "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-platform-secrets + property: git_password {{- if $v._derived.untrustedCA }} - apiVersion: v1 kind: Secret @@ -21,22 +43,53 @@ resources: data: ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} -- apiVersion: v1 - kind: Secret - metadata: - name: gitea-admin-secret - data: - username: "{{ $v.otomi.git.username | b64enc }}" - password: "{{ $v.otomi.git.password | b64enc }}" # DB / app backup resources {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey +{{- end }} +{{- with $v | get "smtp" nil }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-smtp-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-smtp-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .authPassword | toString }}" }}' + data: + - secretKey: authPassword + remoteRef: + key: smtp-secrets + property: auth_password {{- end }} {{- if ne $v.cluster.provider "custom" }} # Application backup resources diff --git a/values/gitea/gitea.gotmpl b/values/gitea/gitea.gotmpl index d7c9bdee21..2f697e0ce2 100644 --- a/values/gitea/gitea.gotmpl +++ b/values/gitea/gitea.gotmpl @@ -63,7 +63,7 @@ gitea: HELO_HOSTNAME: {{ .hello }} FROM: {{ .from }} USER: {{ .auth_username }} - PASSWD: {{ .auth_password | quote }} + PASSWD: placeholder-overridden-by-env MAILER_TYPE: smtp IS_TLS_ENABLED: true SUBJECT_PREFIX: 'Otomi[Gitea]: ' @@ -125,6 +125,13 @@ gitea: secretKeyRef: name: gitea-db-secret key: password + {{- with $v | get "smtp" nil }} + - name: GITEA__MAILER__PASSWD + valueFrom: + secretKeyRef: + name: gitea-smtp-secret + key: password + {{- end }} - name: GITEA__DATABASE__MAX_OPEN_CONNS value: {{ $g.databaseMaxConnections | quote }} - name: GITEA__DATABASE__MAX_IDLE_CONNS diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index 623148ef62..feb9862641 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -33,58 +33,201 @@ resources: issuerRef: name: custom-ca kind: ClusterIssuer -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-admin-password - data: - HARBOR_ADMIN_PASSWORD: "{{ $h.adminPassword | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-admin-password + creationPolicy: Owner + template: + type: Opaque + data: + HARBOR_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: harbor-secrets + property: adminPassword +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-credentials - data: - REGISTRY_PASSWD: "{{ $h.registry.credentials.password | b64enc }}" - REGISTRY_HTPASSWD: "{{ $h.registry.credentials.htpasswd | b64enc }}" -{{- if ne $h.secretKey nil }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_PASSWD: '{{ "{{ .password | toString }}" }}' + REGISTRY_HTPASSWD: '{{ "{{ .htpasswd | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: harbor-secrets + property: registry_credentials_password + - secretKey: htpasswd + remoteRef: + key: harbor-secrets + property: registry_credentials_htpasswd +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-secret-key - data: - secretKey: "{{ $h.secretKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-secret-key + creationPolicy: Owner + template: + type: Opaque + data: + secretKey: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: secretKey + remoteRef: + key: harbor-secrets + property: secretKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-core-secret - data: - secret: "{{ $h.core.secret | b64enc }}" -{{- end }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-secret + creationPolicy: Owner + template: + type: Opaque + data: + secret: '{{ "{{ .coreSecret | toString }}" }}' + data: + - secretKey: coreSecret + remoteRef: + key: harbor-secrets + property: core_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: harbor-core-xsrf-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-xsrf-secret + creationPolicy: Owner + template: + type: Opaque + data: + CSRF_KEY: '{{ "{{ .xsrfKey | toString }}" }}' + data: + - secretKey: xsrfKey + remoteRef: + key: harbor-secrets + property: core_xsrfKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-jobservice-secret - data: - JOBSERVICE_SECRET: "{{ $h.jobservice.secret | default "" | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-jobservice-secret + creationPolicy: Owner + template: + type: Opaque + data: + JOBSERVICE_SECRET: '{{ "{{ .jobserviceSecret | toString }}" }}' + data: + - secretKey: jobserviceSecret + remoteRef: + key: harbor-secrets + property: jobservice_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-http - data: - REGISTRY_HTTP_SECRET: "{{ $h.registry.secret | default "" | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-http + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_HTTP_SECRET: '{{ "{{ .registrySecret | toString }}" }}' + data: + - secretKey: registrySecret + remoteRef: + key: harbor-secrets + property: registry_secret {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: registry-storage-credentials - data: - REGISTRY_STORAGE_S3_ACCESSKEY: "{{ $obj.linode.accessKeyId | b64enc }}" - REGISTRY_STORAGE_S3_SECRETKEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: registry-storage-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_STORAGE_S3_ACCESSKEY: {{ $obj.linode.accessKeyId }} + REGISTRY_STORAGE_S3_SECRETKEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} diff --git a/values/harbor/harbor.gotmpl b/values/harbor/harbor.gotmpl index 24048fef1f..794364489e 100644 --- a/values/harbor/harbor.gotmpl +++ b/values/harbor/harbor.gotmpl @@ -31,7 +31,8 @@ core: resources: {{- $h.resources.core | toYaml | nindent 4 }} existingSecret: harbor-core-secret - xsrfKey: {{ $h | get "core.xsrfKey" nil }} + existingXsrfSecret: harbor-core-xsrf-secret + existingXsrfSecretKey: CSRF_KEY database: maxOpenConns: {{ $h.databaseMaxConnections }} diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 28e89c9963..366b4d6389 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -7,10 +7,42 @@ resources: labels: app.kubernetes.io/component: controller name: {{ $ingress.className }} - {{- if eq $ingress.className $v.ingress.platformClass.className }} + {{- if eq $ingress.className $v.ingress.platformClass.className }} annotations: ingressclass.kubernetes.io/is-default-class: "true" {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" +{{- end }} +# ClusterRole to allow ingress controller to read TLS secrets from all namespaces +{{- range $ingress := $v.ingress.classes }} +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + subjects: + - kind: ServiceAccount + name: ingress-nginx-{{ $ingress.className }} + namespace: ingress {{- end }} \ No newline at end of file diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index eb25208035..65631e6f26 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -72,12 +72,3 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." - - - apiVersion: v1 - kind: Secret - metadata: - name: custom-ca - namespace: cert-manager - data: - tls.crt: {{ $cm.customRootCA | b64enc }} - tls.key: {{ $cm.customRootCAKey | b64enc}} diff --git a/values/keycloak/keycloak-raw.gotmpl b/values/keycloak/keycloak-raw.gotmpl index 3e483fd345..dcf7214cc4 100644 --- a/values/keycloak/keycloak-raw.gotmpl +++ b/values/keycloak/keycloak-raw.gotmpl @@ -1,5 +1,5 @@ {{- $v := .Values }} -{{- $otomiAdmin := "otomi-admin" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} resources: @@ -9,20 +9,50 @@ resources: name: custom-ca data: custom-ca.pem: {{ .Values._derived.caCert | b64enc }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: keycloak-initial-admin namespace: keycloak - data: - password: {{ .Values.otomi.adminPassword | b64enc }} - username: {{ .Values.apps.keycloak.adminUsername | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: keycloak-initial-admin + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + username: {{ $k.adminUsername }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} diff --git a/values/loki/loki-raw.gotmpl b/values/loki/loki-raw.gotmpl index a252ef3dce..3ca9d69809 100644 --- a/values/loki/loki-raw.gotmpl +++ b/values/loki/loki-raw.gotmpl @@ -4,24 +4,69 @@ {{- $obj := $v.obj.provider }} {{- if $v.otomi.isMultitenant }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: reverse-proxy-auth-config - data: - authn.yaml: {{ tpl (readFile "auth-config.gotmpl") (dict "adminPassword" $l.adminPassword "teams" $v.teamConfig) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: reverse-proxy-auth-config + creationPolicy: Owner + template: + type: Opaque + data: + authn.yaml: | + {{ "{{ " }}$adminPassword := .adminPassword | toString{{ " }}" }} + users: + - username: otomi-admin + password: {{ "\"{{ $adminPassword }}\"" }} + orgid: admins + {{- range $id, $team := $v.teamConfig }} + - username: {{ $id }} + password: {{ printf "\"{{ .team_%s_password | toString }}\"" $id }} + orgid: {{ $id }} + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword + {{- range $id, $team := $v.teamConfig }} + - secretKey: team_{{ $id }}_password + remoteRef: + key: team-{{ $id }}-settings-secrets + property: settings_password + {{- end }} {{- if eq $obj.type "linode" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: loki-s3-linode-credentials - type: Opaque - data: - AWS_ACCESS_KEY_ID: "{{ $obj.linode.accessKeyId | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: loki-s3-linode-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: {{ $obj.linode.accessKeyId }} + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} {{- end }} diff --git a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl index bf137b073e..17696000bf 100644 --- a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl @@ -7,15 +7,50 @@ {{- $ingress := $v.ingress.platformClass }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: oauth2-proxy-redis-password + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-redis-password + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: oauth2-proxy-redis-secrets + property: password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: oauth2-proxy-client-access - type: Opaque - data: - client-id: {{ $k.idp.clientID | b64enc }} - client-secret: {{ $k.idp.clientSecret | b64enc }} - cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-client-access + creationPolicy: Owner + template: + type: Opaque + data: + client-id: {{ $k.idp.clientID }} + client-secret: '{{ "{{ .clientSecret | toString }}" }}' + cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) }} + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -204,4 +239,3 @@ resources: - diff --git a/values/oauth2-proxy/oauth2-proxy.gotmpl b/values/oauth2-proxy/oauth2-proxy.gotmpl index e8ab163ce2..7a76a7321c 100644 --- a/values/oauth2-proxy/oauth2-proxy.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy.gotmpl @@ -104,7 +104,8 @@ sessionStorage: standalone: connectionUrl: "redis://oauth2-proxy-redis-ha-haproxy.istio-system.svc.cluster.local:6379" {{- end }} - password: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password + passwordKey: password redis-ha: global: @@ -114,7 +115,7 @@ redis-ha: repository: "{{- $v.otomi.linodeLkeImageRepository }}/docker/redis" {{- end }} enabled: true - redisPassword: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password replicas: {{ $r.replicas }} redis: resources: {{- $r.resources.master | toYaml | nindent 6 }} diff --git a/values/otomi-api/otomi-api-raw.gotmpl b/values/otomi-api/otomi-api-raw.gotmpl new file mode 100644 index 0000000000..51e4f6023b --- /dev/null +++ b/values/otomi-api/otomi-api-raw.gotmpl @@ -0,0 +1,24 @@ +{{- $v := .Values }} +resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: otomi-api-git-credentials + namespace: otomi + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-api-git-credentials + creationPolicy: Owner + template: + type: Opaque + data: + GIT_PASSWORD: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index 6a21424911..52849c396b 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -2,11 +2,9 @@ {{- $c := $v.cluster }} {{- $o := $v.apps | get "otomi-api" }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $sops := $v | get "kms.sops" dict }} {{- $giteaValuesPublicUrl := printf "https://gitea.%s/otomi/values" $v.cluster.domainSuffix }} {{- $git := $v.otomi.git }} {{- $defaultPlatformAdminEmail := printf "platform-admin@%s" $v.cluster.domainSuffix }} -{{- $sopsEnv := tpl (readFile "../../helmfile.d/snippets/sops-env.gotmpl") $sops }} {{- $version := $v.versions | get "api" }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} {{- $coreVersion := $v.otomi.version }} @@ -39,8 +37,7 @@ tools: secrets: GIT_USER: {{ $git.username | quote }} GIT_EMAIL: {{ $git.email | quote }} - GIT_PASSWORD: {{ $git.password | quote }} - {{- $sopsEnv | nindent 2 }} +existingSecret: otomi-api-git-credentials env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }} diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index fff4e04c3b..b74bf5873b 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -1,17 +1,141 @@ {{- $v := .Values }} {{- $p := $v.apps | get "prometheus" }} +{{- $a := $v.apps | get "alertmanager" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} -{{- if or ($p | get "remoteWrite.rwConfig.basicAuth.enabled" false) }} +{{- $receivers := $v | get "alerts.receivers" (list "slack") }} +{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} +{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} +{{- $alertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-admin-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: otomi-admin + admin-password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $k.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword {{- if $p | get "remoteWrite.rwConfig.basicAuth.enabled" false }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: prometheus name: prometheus-remote-write-basic-auth - data: - username: {{ $p.remoteWrite.rwConfig.basicAuth.username | b64enc }} - password: {{ $p.remoteWrite.rwConfig.basicAuth.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: prometheus-remote-write-basic-auth + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $p.remoteWrite.rwConfig.basicAuth.username }} + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: prometheus-secrets + property: remoteWrite_rwConfig_basicAuth_password + {{- end }} + {{- if $a.enabled }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-platform-config + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-platform-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $alertmanagerConfig | nindent 14 }} + data: + {{- if has "slack" $receivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $receivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} {{- end }} -{{- end }} diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index c90b770d14..bc7a7199e5 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -12,9 +12,7 @@ {{- $alertmanagerDomain := printf "alertmanager.%s" $domain }} {{- $prometheusDomain := printf "prometheus.%s" $domain }} {{- $grafanaDomain := printf "grafana.%s" $domain }} -{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} -{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} -{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" ($k | get "idp")) | toString }} +{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" (dict "clientID" ($k | get "idp.clientID" "otomi"))) | toString }} {{- $hasServices := false }} {{- range $teamId, $team := $v.teamConfig }} {{- if gt (len ($team | get "services" list)) 0 }}{{ $hasServices = true }}{{ end }} @@ -199,7 +197,8 @@ alertmanager: priorityClassName: otomi-critical resources: {{- $a.resources.alertmanager | toYaml | nindent 6 }} externalUrl: https://{{ $alertmanagerDomain }} - config: {{- tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 4 }} + useExistingSecret: true + configSecret: alertmanager-platform-config grafana: enabled: {{ $g.enabled }} defaultDashboardsEnabled: false @@ -248,11 +247,25 @@ grafana: {{- end }} basicAuth: true basicAuthUser: otomi-admin - secureJsonData: - basicAuthPassword: {{ $v.apps.loki.adminPassword | quote }} {{- end }} {{- end }} - adminPassword: {{ $g | get "adminPassword" $v.otomi.adminPassword | quote }} + admin: + existingSecret: grafana-admin-secret + userKey: admin-user + passwordKey: admin-password + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: + secretKeyRef: + name: grafana-oidc-secret + key: client_id + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc-secret + key: client_secret + GF_LOKI_BASIC_AUTH_PASSWORD: + secretKeyRef: + name: grafana-loki-datasource-secret + key: password grafana.ini: {{- $grafanaIni | nindent 4 }} server: root_url: https://{{ $grafanaDomain }} diff --git a/versions.yaml b/versions.yaml index d54e4c2e07..cba4e40597 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: main console: main consoleLogin: main -tasks: main +tasks: APL-1476-1 tools: main From 8775f5ac70975976c36bb3d03eb6aa47151205a8 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 20:50:11 +0100 Subject: [PATCH 02/71] feat: create core secrets in apl-secrets namespace --- src/common/git-config.test.ts | 4 +- src/common/git-config.ts | 2 +- src/common/sealed-secrets.test.ts | 52 +++++++++---------- src/common/sealed-secrets.ts | 8 +-- .../external-secrets-raw.gotmpl | 2 +- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 4883b81d94..9ffce19d00 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -353,7 +353,7 @@ describe('git-config', () => { git: { repoUrl: 'https://github.com/org/repo.git', username: 'admin', - password: 'sealed:sealed-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', branch: 'main', email: 'pipeline@cluster.local', }, @@ -361,7 +361,7 @@ describe('git-config', () => { } const result = await getRepo(values, { getK8sSecret: secretMock }) - expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'sealed-secrets') + expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'apl-secrets') expect(result.password).toBe('real-password') expect(result.authenticatedUrl).toContain('real-password') }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 3c18e86333..66cefab43b 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -152,7 +152,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const secret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 0fa226e9b8..cadefb92cc 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -194,10 +194,10 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - // All secrets now go to sealed-secrets namespace + // All secrets now go to apl-secrets namespace const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') expect(harborMapping).toBeDefined() - expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.namespace).toBe('apl-secrets') expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') }) @@ -216,7 +216,7 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) expect(result).toHaveLength(1) - expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].namespace).toBe('apl-secrets') }) it('should serialize users array as single JSON value in users-secrets', async () => { @@ -241,11 +241,11 @@ describe('sealed-secrets', () => { expect(result).toHaveLength(2) const usersMapping = result.find((m) => m.secretName === 'users-secrets') expect(usersMapping).toBeDefined() - expect(usersMapping!.namespace).toBe('sealed-secrets') + expect(usersMapping!.namespace).toBe('apl-secrets') expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) }) - it('should handle teamConfig dynamic paths in sealed-secrets namespace', async () => { + it('should handle teamConfig dynamic paths in apl-secrets namespace', async () => { const secrets = { teamConfig: { 'team-alpha': { someSecret: 'value' }, @@ -258,7 +258,7 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) expect(result).toHaveLength(1) - expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].namespace).toBe('apl-secrets') expect(result[0].secretName).toBe('team-team-alpha-settings-secrets') }) @@ -289,7 +289,7 @@ describe('sealed-secrets', () => { expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') }) - it('should put gitea secrets in sealed-secrets namespace using convention naming', async () => { + it('should put gitea secrets in apl-secrets namespace using convention naming', async () => { const secrets = { apps: { gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, @@ -308,16 +308,16 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - // Harbor should use convention naming in sealed-secrets ns + // Harbor should use convention naming in apl-secrets ns const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') expect(harborMapping).toBeDefined() - expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.namespace).toBe('apl-secrets') expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') - // Gitea should have a gitea-secrets mapping in sealed-secrets ns + // Gitea should have a gitea-secrets mapping in apl-secrets ns const giteaMapping = result.find((m) => m.secretName === 'gitea-secrets') expect(giteaMapping).toBeDefined() - expect(giteaMapping!.namespace).toBe('sealed-secrets') + expect(giteaMapping!.namespace).toBe('apl-secrets') expect(giteaMapping!.data).toHaveProperty('adminPassword', 'gitea-pass') expect(giteaMapping!.data).toHaveProperty('postgresqlPassword', 'pg-pass') }) @@ -326,7 +326,7 @@ describe('sealed-secrets', () => { describe('createSealedSecretManifest', () => { it('should produce correct SealedSecret structure', async () => { const mapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'my-password', secretKey: 'my-secret' }, } @@ -339,18 +339,18 @@ describe('sealed-secrets', () => { expect(result.apiVersion).toBe('bitnami.com/v1alpha1') expect(result.kind).toBe('SealedSecret') expect(result.metadata.name).toBe('harbor-secrets') - expect(result.metadata.namespace).toBe('sealed-secrets') + expect(result.metadata.namespace).toBe('apl-secrets') expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') expect(result.spec.template.type).toBe('Opaque') expect(result.spec.template.metadata.name).toBe('harbor-secrets') - expect(result.spec.template.metadata.namespace).toBe('sealed-secrets') + expect(result.spec.template.metadata.namespace).toBe('apl-secrets') }) it('should call encryptSecretItem for each data key', async () => { const mapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { key1: 'val1', key2: 'val2', key3: 'val3' }, } @@ -361,9 +361,9 @@ describe('sealed-secrets', () => { await createSealedSecretManifest('pem', mapping, deps) expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val1') - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val2') - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val3') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val3') }) }) @@ -376,13 +376,13 @@ describe('sealed-secrets', () => { metadata: { annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, name: 'harbor-secrets', - namespace: 'sealed-secrets', + namespace: 'apl-secrets', }, spec: { encryptedData: { key: 'enc' }, template: { immutable: false, - metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, type: 'Opaque', }, }, @@ -397,9 +397,9 @@ describe('sealed-secrets', () => { await writeSealedSecretManifests(manifests, '/test', deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/sealed-secrets', { recursive: true }) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/apl-secrets', { recursive: true }) expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/manifests/ns/sealed-secrets/harbor-secrets.yaml', + '/test/env/manifests/ns/apl-secrets/harbor-secrets.yaml', 'yaml-content', ) }) @@ -411,7 +411,7 @@ describe('sealed-secrets', () => { apps: { harbor: { adminPassword: 'pass' } }, } const mockMapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'pass' }, } @@ -421,13 +421,13 @@ describe('sealed-secrets', () => { metadata: { annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, name: 'harbor-secrets', - namespace: 'sealed-secrets', + namespace: 'apl-secrets', }, spec: { encryptedData: { adminPassword: 'encrypted' }, template: { immutable: false, - metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, type: 'Opaque', }, }, @@ -462,7 +462,7 @@ describe('sealed-secrets', () => { apps: { harbor: { adminPassword: 'pass' } }, } const mockMapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'pass' }, } diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 4c364a4946..33c37c95a5 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -240,21 +240,21 @@ export const createSealedSecretsKeySecret = async ( /** * Resolve the namespace for a given secret path. - * All core secrets go to 'sealed-secrets' namespace for ESO access. + * All core secrets go to 'apl-secrets' namespace for ESO access. * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. */ const resolveNamespace = (secretPath: string): string | undefined => { // Check for teamConfig dynamic paths const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) if (teamMatch) { - return 'sealed-secrets' + return 'apl-secrets' } // Check if this path matches any known prefix const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) for (const prefix of sortedKeys) { if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return 'sealed-secrets' + return 'apl-secrets' } } @@ -355,7 +355,7 @@ export const buildSecretToNamespaceMap = async ( if (secretPath === 'users') { const usersData = secrets.users if (Array.isArray(usersData) && usersData.length > 0) { - const namespace = 'sealed-secrets' + const namespace = 'apl-secrets' const secretName = 'users-secrets' const groupKey = `${namespace}/${secretName}` if (!groupMap.has(groupKey)) { diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl index c543f54b1e..c1b95b3111 100644 --- a/values/external-secrets/external-secrets-raw.gotmpl +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -33,7 +33,7 @@ resources: spec: provider: kubernetes: - remoteNamespace: sealed-secrets + remoteNamespace: apl-secrets server: url: "https://kubernetes.default.svc" caProvider: From 45a18611717f0904b9ae69c52c3d72a11f1a0853 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 20:53:27 +0100 Subject: [PATCH 03/71] fix: add default value for the existingSecret --- charts/otomi-api/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index 8cdfe5787a..f4ddb4695e 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -24,6 +24,8 @@ serviceAccount: imagePullSecrets: {} +existingSecret: "" + rbac: # Specifies whether rbac should be set up create: true From cd503a9a1e6481ade53a8cf89232854c9d45cd9c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 22:31:39 +0100 Subject: [PATCH 04/71] fix: namespace changes --- charts/apl-operator/templates/deployment.yaml | 2 ++ src/cmd/commit.ts | 4 ++-- src/cmd/migrate.ts | 2 +- values/apl-operator/apl-operator.gotmpl | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index c4f5011207..7683978046 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -44,8 +44,10 @@ spec: envFrom: - secretRef: name: apl-sops-secrets + optional: true - secretRef: name: apl-git-credentials + optional: true resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 96f427920e..a497aa8290 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -176,7 +176,7 @@ export async function initialSetupData(): Promise { if (!hasExternalIDP) { // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) - const usersSecret = await getK8sSecret('users-secrets', 'sealed-secrets') + const usersSecret = await getK8sSecret('users-secrets', 'apl-secrets') let platformAdminPassword = '' if (usersSecret?.usersJson) { // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string @@ -195,7 +195,7 @@ export async function initialSetupData(): Promise { } } else { // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'apl-secrets') const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 0ad054f1f0..f5c9cf5121 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -712,7 +712,7 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - const giteaSecrets = await getK8sSecret('gitea-secrets', 'sealed-secrets') + const giteaSecrets = await getK8sSecret('gitea-secrets', 'apl-secrets') const resolvedGitea = { adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 03463b273a..1657090d50 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -18,3 +18,9 @@ imagePullSecrets: resources: {{- toYaml $o.resources.operator | nindent 2 }} kms: {{- $kms | toYaml | nindent 2 }} + +git: + repoUrl: {{ $v.otomi.git.repoUrl | quote }} + branch: {{ $v.otomi.git.branch | default "main" | quote }} + email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} + username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} From edc8ffdc540aeac0d4ce3bcde13237815e8e48b6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 23:35:34 +0100 Subject: [PATCH 05/71] test: sealed secrets with eso --- src/operator/installer.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 5bd01f228b..77aeca38a8 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -154,12 +154,14 @@ export class Installer { const agePrivateKey = values?.kms?.sops?.age?.privateKey // Ensure apl-git-credentials secret + // Only recreate if credentials are missing AND values has the password available + // (password may be undefined when secrets are stripped from disk and managed by SealedSecrets + ESO) const credentials = await getGitCredentials() - if (!credentials) { + if (!credentials && otomiGit?.username && otomiGit?.password) { this.d.info('Recreating apl-git-credentials secret') await createUpdateGenericSecret(k8s.core(), GIT_CONFIG_SECRET_NAME, GIT_CONFIG_NAMESPACE, { - username: otomiGit?.username, - password: otomiGit?.password, + username: otomiGit.username, + password: otomiGit.password, }) } From 0a4cc5a0fd53ec4bd8b9ef4b0db1cde876cafbc9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 09:12:04 +0100 Subject: [PATCH 06/71] fix: merge conflicts/changes --- src/common/git-config.test.ts | 9 + src/common/git-config.ts | 5 + src/operator/installer.test.ts | 253 +++--------------------- src/operator/installer.ts | 102 +++------- src/operator/main.ts | 22 ++- values/apl-operator/apl-operator.gotmpl | 1 + 6 files changed, 79 insertions(+), 313 deletions(-) diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 9ffce19d00..ef0c5a2ab2 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -67,6 +67,15 @@ describe('git-config', () => { const result = await getGitCredentials() expect(result).toBeUndefined() }) + + it('should return undefined when password is a sealed-secret placeholder', async () => { + mockGetK8sSecret.mockResolvedValue({ + username: 'admin', + password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + }) + const result = await getGitCredentials() + expect(result).toBeUndefined() + }) }) describe('getOldGitCredentials', () => { diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 66cefab43b..cfa824d535 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -40,6 +40,11 @@ export async function getGitCredentials(): Promise { return undefined } + // Reject unresolved sealed-secret placeholders (e.g. during first deploy before secrets are decrypted) + if (typeof secretData.password === 'string' && secretData.password.startsWith('sealed:')) { + return undefined + } + return { username: secretData.username, password: secretData.password, diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 02cb411730..96a09eff0e 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -1,5 +1,4 @@ import * as gitConfig from '../common/git-config' -import * as hf from '../common/hf' import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' @@ -14,21 +13,6 @@ jest.mock('zx', () => ({ $: (...args: any[]) => mockZx(...args), })) -jest.mock('../common/envalid', () => ({ - env: { - GIT_PROTOCOL: 'http', - GIT_URL: 'gitea-http.gitea.svc.cluster.local', - GIT_PORT: '3000', - }, -})) - -jest.mock('./validators', () => ({ - operatorEnv: { - GIT_ORG: 'otomi', - GIT_REPO: 'values', - }, -})) - jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -39,7 +23,6 @@ jest.mock('../common/debug', () => ({ })) jest.mock('../common/k8s', () => ({ - deletePendingHelmReleases: jest.fn(), getK8sConfigMap: jest.fn(), getK8sSecret: jest.fn(), createUpdateConfigMap: jest.fn(), @@ -49,17 +32,8 @@ jest.mock('../common/k8s', () => ({ }, })) -jest.mock('../common/hf', () => ({ - hfValues: jest.fn(), -})) - jest.mock('../common/git-config', () => ({ - getGitCredentials: jest.fn().mockResolvedValue(undefined), - getGitConfigData: jest.fn().mockResolvedValue(undefined), - getStoredGitRepoConfig: jest.fn().mockResolvedValue(undefined), - setGitConfig: jest.fn().mockResolvedValue(undefined), - GIT_CONFIG_SECRET_NAME: 'apl-git-credentials', - GIT_CONFIG_NAMESPACE: 'apl-operator', + getStoredGitRepoConfig: jest.fn(), })) jest.mock('./utils', () => ({ @@ -173,7 +147,7 @@ describe('Installer', () => { }), ) - // Verify failed status was recorded + // Verify failed status was recorded with error message expect(k8s.createUpdateConfigMap).toHaveBeenCalledWith( mockCoreApi, 'apl-installation-status', @@ -181,6 +155,7 @@ describe('Installer', () => { expect.objectContaining({ status: 'failed', attempt: '1', + error: 'Install failed', }), ) @@ -265,8 +240,15 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - // getK8sSecret returns credentials for git verification - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // getStoredGitRepoConfig returns valid config + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) // git ls-remote succeeds (main branch exists) mockZx.mockReturnValue({ nothrow: jest.fn().mockReturnValue({ @@ -285,7 +267,14 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) // git ls-remote fails (main branch does not exist) mockZx.mockReturnValue({ nothrow: jest.fn().mockReturnValue({ @@ -335,8 +324,8 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - // getK8sSecret throws (cluster issues) - ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + // getStoredGitRepoConfig throws (cluster issues) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('connection refused')) const isInstalled = await installer.isInstalled() @@ -346,10 +335,10 @@ describe('Installer', () => { }) describe('setEnvAndCreateSecrets', () => { - test('should use existing credentials from apl-git-credentials secret when available', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) // apl-sops-secrets + test('should use existing SOPS key from secret', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) - const result = await installer.setEnvAndCreateSecrets() + await installer.setEnvAndCreateSecrets() expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') @@ -364,196 +353,4 @@ describe('Installer', () => { expect(process.env.SOPS_AGE_KEY).toBe('') }) }) - - describe('ensureSecretsAndConfig', () => { - const mockValues = { - otomi: { - git: { - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - username: 'admin', - password: 's3cret', - }, - }, - kms: { - sops: { - age: { - privateKey: 'AGE-SECRET-KEY-1234', - }, - }, - }, - } - - test('should skip when hfValues returns undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.getGitCredentials).not.toHaveBeenCalled() - expect(k8s.getK8sSecret).not.toHaveBeenCalled() - expect(gitConfig.getGitConfigData).not.toHaveBeenCalled() - }) - - test('should not recreate resources when all exist', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() - expect(gitConfig.setGitConfig).not.toHaveBeenCalled() - }) - - test('should recreate git credentials when missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) - }) - - test('should recreate sops secret when missing and age key exists in values', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) - }) - - test('should not recreate sops secret when missing but no age key in values', async () => { - const valuesWithoutSops = { - ...mockValues, - kms: { sops: { age: {} } }, - } - ;(hf.hfValues as jest.Mock).mockResolvedValue(valuesWithoutSops) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() - }) - - test('should recreate git config when repoUrl is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when branch is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when email is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when configData is undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate all resources when all are missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 77aeca38a8..682c2dabf1 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,18 +1,10 @@ import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' -import { - getGitConfigData, - getGitCredentials, - GIT_CONFIG_NAMESPACE, - GIT_CONFIG_SECRET_NAME, - setGitConfig, -} from '../common/git-config' -import { hfValues } from '../common/hf' -import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { getStoredGitRepoConfig } from '../common/git-config' +import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' -import { operatorEnv } from './validators' export class Installer { private d = terminal('operator:installer') @@ -45,17 +37,14 @@ export class Installer { private async verifyGitRepoHasMainBranch(): Promise { try { - // Get credentials from K8s secret (created by Helm at deploy time) - const creds = await getK8sSecret('gitea-credentials', 'apl-operator') - const username = creds?.GIT_USERNAME ?? 'otomi-admin' - const password = creds?.GIT_PASSWORD ?? '' - const repoUrl = `${process.env.GIT_PROTOCOL}://${username}:${password}@${process.env.GIT_URL}:${process.env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` - const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + const gitConfig = await getStoredGitRepoConfig() + if (!gitConfig) return true // Can't verify without config, assume fine + const result = await $`git ls-remote --exit-code --heads ${gitConfig.authenticatedUrl} main`.nothrow().quiet() return result.exitCode === 0 } catch { // If we can't check (e.g. gitea not ready yet), assume it's fine // The operator will detect the issue later during git polling - this.d.warn('Could not verify git repo - gitea may not be ready yet') + this.d.warn('Could not verify git repo - may not be ready yet') return true } } @@ -87,13 +76,14 @@ export class Installer { // Run the installation sequence await this.updateInstallationStatus('in-progress', attemptNumber) await this.aplOps.install() - await this.ensureSecretsAndConfig() await this.updateInstallationStatus('completed', attemptNumber) return } catch (error) { - await this.updateInstallationStatus('failed', attemptNumber) - this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`, getErrorMessage(error)) + const errorMessage = getErrorMessage(error) + this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) + await this.updateInstallationStatus('failed', attemptNumber, errorMessage) + this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) // Wait 1 second before retrying await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -109,12 +99,14 @@ export class Installer { return status } - private async updateInstallationStatus(status: string, attempt: number): Promise { + private async updateInstallationStatus(status: string, attempt: number, error?: string): Promise { try { const data = { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), + // Always include error field to prevent stale values from StrategicMergePatch + error: error ?? '', } await createUpdateConfigMap(k8s.core(), 'apl-installation-status', 'apl-operator', data) @@ -129,60 +121,20 @@ export class Installer { } private async setupSopsEnvironment() { - const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - - if (aplSopsSecret?.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY - this.d.debug('Using existing sops credentials from secret') - } else { - // SOPS is no longer used (replaced by SealedSecrets + ESO). - // Skip hfValues() call which requires the git repo that may not exist yet. - this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') - } - } - - // public for testing. This method should only be used if you are certain there are values locally. - async ensureSecretsAndConfig(): Promise { - this.d.info('Verifying secrets and config after installation') - const values = (await hfValues()) as Record - if (!values) { - this.d.warn('Could not retrieve hfValues, skipping secrets/config verification') - return - } - - const otomiGit = values?.otomi?.git - const agePrivateKey = values?.kms?.sops?.age?.privateKey - - // Ensure apl-git-credentials secret - // Only recreate if credentials are missing AND values has the password available - // (password may be undefined when secrets are stripped from disk and managed by SealedSecrets + ESO) - const credentials = await getGitCredentials() - if (!credentials && otomiGit?.username && otomiGit?.password) { - this.d.info('Recreating apl-git-credentials secret') - await createUpdateGenericSecret(k8s.core(), GIT_CONFIG_SECRET_NAME, GIT_CONFIG_NAMESPACE, { - username: otomiGit.username, - password: otomiGit.password, - }) - } - - // Ensure apl-sops-secrets secret - const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret?.SOPS_AGE_KEY && agePrivateKey) { - this.d.info('Recreating apl-sops-secrets secret') - await createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', GIT_CONFIG_NAMESPACE, { - SOPS_AGE_KEY: agePrivateKey, - }) - } - - // Ensure apl-git-config configmap - const configData = await getGitConfigData() - if (!configData?.repoUrl || !configData?.branch || !configData?.email) { - this.d.info('Recreating apl-git-config configmap') - await setGitConfig({ - repoUrl: otomiGit?.repoUrl, - branch: otomiGit?.branch, - email: otomiGit?.email, - }) + try { + const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') + + if (aplSopsSecret?.SOPS_AGE_KEY) { + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.debug('Using existing sops credentials from secret') + } else { + // SOPS is no longer used (replaced by SealedSecrets + ESO). + // Skip hfValues() call which requires the git repo that may not exist yet. + this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') + } + } catch (error) { + this.d.error('Failed to retrieve sops credentials:', getErrorMessage(error)) + throw error } } } diff --git a/src/operator/main.ts b/src/operator/main.ts index bc8e897e60..e601f85fbe 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -1,17 +1,17 @@ import * as dotenv from 'dotenv' -import { terminal } from '../common/debug' -import { AplOperator, AplOperatorConfig } from './apl-operator' -import { Installer } from './installer' -import { operatorEnv } from './validators' -import { env } from '../common/envalid' import fs from 'fs' +import process from 'node:process' import path from 'path' +import { runTraceCollectionLoop } from '../cmd/traces' +import { terminal } from '../common/debug' +import { env } from '../common/envalid' +import { getStoredGitRepoConfig } from '../common/git-config' import { AplOperations } from './apl-operations' -import { getErrorMessage } from './utils' +import { AplOperator, AplOperatorConfig } from './apl-operator' import { GitRepository } from './git-repository' -import { getStoredGitRepoConfig } from '../common/git-config' -import process from 'node:process' -import { runTraceCollectionLoop } from '../cmd/traces' +import { Installer } from './installer' +import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' dotenv.config() @@ -78,13 +78,15 @@ async function main(): Promise { await installer.reconcileInstall() } + // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) + await installer.setEnvAndCreateSecrets() + // Start trace collection in background (runs for 30 minutes from ConfigMap creation) runTraceCollectionLoop().catch((error) => { d.warn('Trace collection loop failed:', getErrorMessage(error)) }) // Phase 2: Set environment variables and start operator for GitOps operations - // await installer.setEnvAndCreateSecrets() const config = await loadConfig(aplOps) const operator = new AplOperator(config) handleTerminationSignals(operator) diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 1657090d50..56acfc091b 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,3 +24,4 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} + password: {{ $v.otomi.git.password | default "" | quote }} From c5b029f8cb5b804ed82b819a9d6ddee13970e969 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:47:44 +0100 Subject: [PATCH 07/71] fix: merge conflicts/changes --- helmfile.d/helmfile-04.init.yaml.gotmpl | 1 + src/common/k8s.test.ts | 6 +++--- src/common/k8s.ts | 16 +++++++++------- src/operator/installer.test.ts | 1 + src/operator/installer.ts | 13 ++++++++++--- values/apl-operator/apl-operator-raw.gotmpl | 2 +- values/apl-operator/apl-operator.gotmpl | 2 +- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/helmfile.d/helmfile-04.init.yaml.gotmpl b/helmfile.d/helmfile-04.init.yaml.gotmpl index df2adc1e35..6a3d9121f7 100644 --- a/helmfile.d/helmfile-04.init.yaml.gotmpl +++ b/helmfile.d/helmfile-04.init.yaml.gotmpl @@ -30,6 +30,7 @@ releases: namespace: apl-operator labels: pkg: apl-operator + app: core <<: *raw - name: otomi-operator installed: true diff --git a/src/common/k8s.test.ts b/src/common/k8s.test.ts index bc35c28693..99d454b1d4 100644 --- a/src/common/k8s.test.ts +++ b/src/common/k8s.test.ts @@ -14,8 +14,8 @@ import { V1StatefulSet, V1Status, } from '@kubernetes/client-node' -import { X509Certificate } from 'crypto' import retry from 'async-retry' +import { X509Certificate } from 'crypto' import { ARGOCD_APP_PARAMS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -733,8 +733,8 @@ describe('helm operations in progress check', () => { await k8s.deletePendingHelmReleases() expect(mockGetPendingHelmReleases).toHaveBeenCalled() - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(1, 'release-1', 'ns-1') - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2') + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(2, 'release-1', 'ns-1', 2) + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2', 1) }) }) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index e701de4f7b..dcfc943300 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -16,9 +16,9 @@ import { V1Status, } from '@kubernetes/client-node' import retry, { Options } from 'async-retry' +import { X509Certificate } from 'crypto' import { AnyAaaaRecord, AnyARecord } from 'dns' import { resolveAny } from 'dns/promises' -import { X509Certificate } from 'crypto' import { access, mkdir, writeFile } from 'fs/promises' import { isEmpty, isEqual, map, mapValues } from 'lodash' import { dirname, join } from 'path' @@ -149,12 +149,12 @@ export const getK8sSecret = async (name: string, namespace: string): Promise { +export const deleteSecretForHelmRelease = async (releaseName: string, namespace: string, revision = 1) => { const d = terminal('common:k8s:deleteSecretForHelmRelease') - d.info(`Deleting secret for Helm release ${releaseName} in namespace ${namespace}`) + d.info(`Deleting secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) try { - await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v1`, namespace }) - d.debug(`Deleted secret for Helm release ${releaseName} in namespace ${namespace}`) + await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v${revision}`, namespace }) + d.debug(`Deleted secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) } catch (error) { if (error?.response?.statusCode !== 404) { throw error @@ -194,9 +194,11 @@ export const deletePendingHelmReleases = async (): Promise => { const d = terminal(`common:k8s:deletePendingHelmReleases`) const pendingHelmReleases = await getPendingHelmReleases() if (pendingHelmReleases.length > 0) { - d.info(`Pending Helm operations detected for releases: ${pendingHelmReleases.join(', ')}. removing secrets...`) + d.info( + `Pending Helm operations detected for releases: ${pendingHelmReleases.map((r) => `${r.namespace}/${r.name}:v${r.revision}`).join(', ')}. removing secrets...`, + ) for (const release of pendingHelmReleases) { - await deleteSecretForHelmRelease(release.name, release.namespace) + await deleteSecretForHelmRelease(release.name, release.namespace, release.revision) } } } diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 96a09eff0e..22dece724d 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -27,6 +27,7 @@ jest.mock('../common/k8s', () => ({ getK8sSecret: jest.fn(), createUpdateConfigMap: jest.fn(), createUpdateGenericSecret: jest.fn(), + deletePendingHelmReleases: jest.fn().mockResolvedValue(undefined), k8s: { core: jest.fn(), }, diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 682c2dabf1..e55f28edfd 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -2,7 +2,7 @@ import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -83,9 +83,16 @@ export class Installer { const errorMessage = getErrorMessage(error) this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) await this.updateInstallationStatus('failed', attemptNumber, errorMessage) - this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) - // Wait 1 second before retrying + // Clean up stuck Helm releases (e.g. pending-install, pending-upgrade) + // so the next retry can proceed without "another operation is in progress" errors + try { + await deletePendingHelmReleases() + } catch (cleanupError) { + this.d.warn('Failed to clean up pending Helm releases:', getErrorMessage(cleanupError)) + } + + this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) await new Promise((resolve) => setTimeout(resolve, 1000)) } } diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 1ad5c4a39b..bd18d18d0e 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -16,7 +16,7 @@ resources: template: type: Opaque data: - GIT_USERNAME: {{ $v.otomi.git.username | default "otomi-admin" }} + GIT_USERNAME: {{ $v.otomi.git | get "username" "otomi-admin" }} GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' data: - secretKey: git_password diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 56acfc091b..bfe276488b 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,4 +24,4 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} - password: {{ $v.otomi.git.password | default "" | quote }} + password: {{ $v.otomi.git | get "password" "" | quote }} From 4fd4dfefccf19aadb0f762e316f587fd1b4be3ff Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:29:35 +0100 Subject: [PATCH 08/71] fix: merge conflicts/changes --- src/cmd/bootstrap.test.ts | 31 +++++++++++++++++++++++++++++++ src/cmd/bootstrap.ts | 5 +++++ 2 files changed, 36 insertions(+) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 6cfb0f22a1..0a1b004fc7 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -449,6 +449,37 @@ describe('Bootstrapping values', () => { merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), ) }) + it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { + // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) + const storedProcessedUsers = [ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ] + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.generateSecrets.mockReturnValue({}) + deps.createCustomCA.mockReturnValue({}) + // getUsers returns the stored processed users (no isPlatformAdmin flag) + deps.getUsers.mockReturnValue(storedProcessedUsers) + + const result = await processValues(deps) + + // Groups should be preserved from existing data, not reset to [] + expect(result.allSecrets.users).toEqual([ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ]) + }) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index f5d4c13f60..99da59dac5 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -346,6 +346,11 @@ export const processValues = async ( if (user.isPlatformAdmin) groups.push('platform-admin') if (user.isTeamAdmin) groups.push('team-admin') for (const team of user.teams || []) groups.push(`team-${team}`) + // Preserve existing groups when boolean flags are absent (e.g., user recovered + // from stored secrets which uses the processed format without isPlatformAdmin/isTeamAdmin) + if (groups.length === 0 && Array.isArray(user.groups) && user.groups.length > 0) { + groups.push(...(user.groups as string[])) + } return { email: user.email, firstName: user.firstName, From 0d8ca8478eba61ba91e01d6b1f3bffb3f24365db Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:00:55 +0100 Subject: [PATCH 09/71] feat: waiting for sealed secrets --- src/cmd/install.ts | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 8a221696cf..1416f0de83 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -89,31 +89,35 @@ const waitForSealedSecrets = async ( } d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) - const start = Date.now() - - while (Date.now() - start < timeoutMs) { - const pending: string[] = [] - for (const { namespace, secretName } of secretsToWait.values()) { - try { - const secret = await deps.getK8sSecret(secretName, namespace) - if (!secret) { + + await retry( + async () => { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait.values()) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { pending.push(`${namespace}/${secretName}`) } - } catch { - pending.push(`${namespace}/${secretName}`) } - } - if (pending.length === 0) { - d.info('All sealed secrets have been decrypted') - return - } - - d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) - await new Promise((resolve) => setTimeout(resolve, intervalMs)) - } + if (pending.length > 0) { + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + throw new Error(`Sealed secrets not yet decrypted: ${pending.join(', ')}`) + } - throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) + d.info('All sealed secrets have been decrypted') + }, + { + retries: Math.ceil(timeoutMs / intervalMs), + minTimeout: intervalMs, + maxTimeout: intervalMs, + factor: 1, + }, + ) } export const installAll = async () => { From 431bc0fe84bdae144df4f6cd5876cb81a4a02d67 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:11:06 +0100 Subject: [PATCH 10/71] feat: move function to k8s.ts --- src/common/k8s.ts | 26 ++++++++++++++++++++++++++ src/common/sealed-secrets.ts | 27 +-------------------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index dcfc943300..f6e0c6726c 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -962,6 +962,32 @@ export async function waitForCRD(crdName: string, timeoutSeconds: number = 60): } } +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:k8s:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + export async function getSealedSecretsPEM(): Promise { const d = terminal('common:k8s:getSealedSecretsPEM') const namespace = 'sealed-secrets' diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 33c37c95a5..84d12fd242 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -6,6 +6,7 @@ import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { terminal } from 'src/common/debug' +import { ensureNamespaceExists } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' import { $ } from 'zx' @@ -25,32 +26,6 @@ export function stripAllSecrets(values: Record, secretPaths: string return stripped } -/** - * Ensure a namespace exists. If it doesn't exist, create it with proper labels. - * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. - */ -export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { - const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) - - // Check if namespace already exists - const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() - if (existingNs.exitCode === 0) { - d.debug(`Namespace ${namespace} already exists`) - return - } - - // Create namespace with proper label - d.info(`Creating namespace ${namespace}`) - const nsYaml = `apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} - labels: - name: ${namespace}` - - await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() -} - export interface SecretMapping { namespace: string secretName: string From 8ac4c9696ac9b8813683104ea7bae42b4edabe84 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:04:28 +0100 Subject: [PATCH 11/71] feat: use kubernetes package instead of kubectl --- src/common/k8s.ts | 35 ++--- src/common/sealed-secrets.test.ts | 67 ++++----- src/common/sealed-secrets.ts | 220 ++++++++++++++++++++---------- 3 files changed, 189 insertions(+), 133 deletions(-) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index f6e0c6726c..6f37001e5c 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -966,26 +966,27 @@ export async function waitForCRD(crdName: string, timeoutSeconds: number = 60): * Ensure a namespace exists. If it doesn't exist, create it with proper labels. * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. */ -export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { - const d = deps.terminal(`common:k8s:ensureNamespaceExists`) +export const ensureNamespaceExists = async (namespace: string): Promise => { + const d = terminal(`common:k8s:ensureNamespaceExists`) - // Check if namespace already exists - const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() - if (existingNs.exitCode === 0) { + try { + await k8s.core().readNamespace({ name: namespace }) d.debug(`Namespace ${namespace} already exists`) - return + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info(`Creating namespace ${namespace}`) + await k8s.core().createNamespace({ + body: { + metadata: { + name: namespace, + labels: { name: namespace }, + }, + }, + }) + } else { + throw error + } } - - // Create namespace with proper label - d.info(`Creating namespace ${namespace}`) - const nsYaml = `apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} - labels: - name: ${namespace}` - - await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() } export async function getSealedSecretsPEM(): Promise { diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index cadefb92cc..7dbd238273 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -19,12 +19,25 @@ jest.mock('@linode/kubeseal-encrypt', () => ({ encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), })) -jest.mock('zx', () => ({ - $: jest.fn().mockReturnValue({ - nothrow: jest.fn().mockReturnValue({ - quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), +jest.mock('src/common/k8s', () => ({ + getK8sSecret: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), + b64enc: jest.fn((v: string) => Buffer.from(v).toString('base64')), + k8s: { + core: jest.fn().mockReturnValue({ + createNamespacedSecret: jest.fn().mockResolvedValue({}), }), - }), + app: jest.fn().mockReturnValue({ + patchNamespacedDeployment: jest.fn().mockResolvedValue({}), + readNamespacedDeployment: jest + .fn() + .mockResolvedValue({ spec: { replicas: 1 }, status: { updatedReplicas: 1, availableReplicas: 1 } }), + }), + custom: jest.fn().mockReturnValue({ + createNamespacedCustomObject: jest.fn().mockResolvedValue({}), + patchNamespacedCustomObject: jest.fn().mockResolvedValue({}), + }), + }, })) jest.mock('src/common/envalid', () => ({ @@ -125,59 +138,27 @@ describe('sealed-secrets', () => { describe('createSealedSecretsKeySecret', () => { it('should create secret if it does not exist', async () => { - const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) - const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) - // First call (namespace): success, Second call (check exists): not found (exitCode 1) - // Third call (create): success, Fourth call (label): success - const mock$ = jest - .fn() - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // namespace - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), - }) // check exists - NOT FOUND - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // create - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // label + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) const deps = { - $: mock$ as any, + getK8sSecret: mockGetK8sSecret, terminal, - writeFile: jest.fn(), - mkdir: jest.fn(), } await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) - expect(mock$).toHaveBeenCalledTimes(4) - expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') - expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') }) it('should skip creation if secret already exists', async () => { - const mock$ = jest - .fn() - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // namespace - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // check exists - FOUND + const mockGetK8sSecret = jest.fn().mockResolvedValue({ 'tls.crt': 'existing-cert' }) const deps = { - $: mock$ as any, + getK8sSecret: mockGetK8sSecret, terminal, - writeFile: jest.fn(), - mkdir: jest.fn(), } await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) - // Should only call namespace and check exists, not create or label - expect(mock$).toHaveBeenCalledTimes(2) - expect(deps.writeFile).not.toHaveBeenCalled() + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 84d12fd242..271281b76a 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -1,3 +1,4 @@ +import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' import { X509Certificate } from 'crypto' import { existsSync } from 'fs' @@ -6,10 +7,10 @@ import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { terminal } from 'src/common/debug' -import { ensureNamespaceExists } from 'src/common/k8s' +import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' -import { $ } from 'zx' +import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' @@ -136,28 +137,30 @@ export const getPemFromCertificate = (certificate: string): string => { /** * Get the existing sealed-secrets certificate from the cluster if it exists. * Returns the certificate PEM string or undefined if not found. + * Note: Uses k8s client directly instead of getK8sSecret() because PEM certificates + * are corrupted by the YAML parse step in getK8sSecret(). */ -export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { +export const getExistingSealedSecretsCert = async (deps = { k8s, terminal }): Promise => { const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) - const result = - await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` - .nothrow() - .quiet() - - if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { - d.info('No existing sealed-secrets-key found') - return undefined - } - try { - const certBase64 = result.stdout.replace(/'/g, '') - const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + const secret = await deps.k8s.core().readNamespacedSecret({ + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + }) + if (!secret?.data?.['tls.crt']) { + d.info('No existing sealed-secrets-key found') + return undefined + } + d.info('Found existing sealed-secrets-key certificate') - return cert - } catch { - d.warn('Failed to decode existing certificate') - return undefined + return Buffer.from(secret.data['tls.crt'], 'base64').toString('utf-8') + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info('No existing sealed-secrets-key found') + return undefined + } + throw error } } @@ -169,46 +172,38 @@ export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Prom export const createSealedSecretsKeySecret = async ( certificate: string, privateKey: string, - deps = { $, terminal, writeFile, mkdir }, + deps = { getK8sSecret, terminal }, ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - // Create namespace if it doesn't exist - await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists('sealed-secrets') // Check if secret already exists - const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() - if (existingSecret.exitCode === 0) { + const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') + if (existing) { d.info('sealed-secrets-key already exists, skipping creation') return } d.info('Creating sealed-secrets TLS secret') - // Write temp files for kubectl create secret tls - const tmpDir = '/tmp/sealed-secrets-bootstrap' - await deps.mkdir(tmpDir, { recursive: true }) - const certPath = `${tmpDir}/tls.crt` - const keyPath = `${tmpDir}/tls.key` - await deps.writeFile(certPath, certificate) - await deps.writeFile(keyPath, privateKey) - - // Create the TLS secret (only if it doesn't exist) - const result = - await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` - .nothrow() - .quiet() - if (result.exitCode !== 0) { - d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) - return - } - - // Label the secret so the controller picks it up - const labelResult = - await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` - .nothrow() - .quiet() - if (labelResult.stderr) d.error(labelResult.stderr) + await k8s.core().createNamespacedSecret({ + namespace: 'sealed-secrets', + body: { + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { + 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + }, + }, + type: 'kubernetes.io/tls', + data: { + 'tls.crt': b64enc(certificate), + 'tls.key': b64enc(privateKey), + }, + }, + }) d.info('Created sealed-secrets TLS secret with key label') } @@ -436,7 +431,7 @@ export const writeSealedSecretManifests = async ( */ export const applySealedSecretManifests = async ( manifests: SealedSecretManifest[], - deps = { $, terminal, objectToYaml }, + deps = { terminal }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) @@ -452,14 +447,34 @@ export const applySealedSecretManifests = async ( // Ensure namespaces exist and apply manifests for (const [namespace, nsManifests] of byNamespace) { - await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists(namespace) for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) - const yaml = deps.objectToYaml(manifest) - const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() - if (result.exitCode !== 0) { - d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + try { + await k8s.custom().createNamespacedCustomObject({ + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + body: manifest, + }) + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + } else { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) + } } } } @@ -473,7 +488,7 @@ export const applySealedSecretManifests = async ( */ export const applySealedSecretManifestsFromDir = async ( envDir: string, - deps = { $, terminal, readdir, readFile, existsSync }, + deps = { terminal, readdir, readFile, existsSync }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/ns') @@ -494,8 +509,7 @@ export const applySealedSecretManifestsFromDir = async ( const namespace = nsEntry.name const nsDir = join(manifestsDir, namespace) - // Ensure namespace exists with proper labels - await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists(namespace) // Read all YAML files in the namespace directory const files = await deps.readdir(nsDir) @@ -504,11 +518,39 @@ export const applySealedSecretManifestsFromDir = async ( const filePath = join(nsDir, file) d.info(`Applying SealedSecret from ${filePath}`) - const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() - if (result.exitCode !== 0) { - d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) - } else { - appliedCount += 1 + try { + const content = await deps.readFile(filePath, 'utf-8') + const manifest = parseYaml(content) as SealedSecretManifest + + try { + await k8s.custom().createNamespacedCustomObject({ + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + body: manifest, + }) + appliedCount += 1 + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + appliedCount += 1 + } else { + d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) + } + } + } catch (parseError) { + d.error(`Failed to parse SealedSecret from ${filePath}: ${parseError}`) } } } @@ -521,25 +563,57 @@ export const applySealedSecretManifestsFromDir = async ( * This is needed because if the controller starts before the sealed-secrets-key secret exists, * it will generate its own key. Restarting forces it to pick up the existing key. */ -export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { +export const restartSealedSecretsController = async (deps = { terminal }): Promise => { const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) d.info('Restarting sealed-secrets controller to ensure correct key is used') - const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() - if (result.exitCode !== 0) { - d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + try { + await k8s.app().patchNamespacedDeployment( + { + name: 'sealed-secrets', + namespace: 'sealed-secrets', + body: { + spec: { + template: { + metadata: { + annotations: { + 'kubectl.kubernetes.io/restartedAt': new Date().toISOString(), + }, + }, + }, + }, + }, + }, + setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), + ) + } catch (error) { + d.warn(`Failed to restart sealed-secrets controller: ${error}`) return } d.info('Waiting for sealed-secrets controller rollout') - const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` - .nothrow() - .quiet() - if (waitResult.exitCode !== 0) { - d.warn(`Rollout status check failed: ${waitResult.stderr}`) - } else { - d.info('Sealed-secrets controller restarted successfully') + const timeoutMs = 120000 + const intervalMs = 3000 + const start = Date.now() + while (Date.now() - start < timeoutMs) { + try { + const deployment = await k8s.app().readNamespacedDeployment({ + name: 'sealed-secrets', + namespace: 'sealed-secrets', + }) + const desired = deployment.spec?.replicas ?? 1 + const updated = deployment.status?.updatedReplicas ?? 0 + const available = deployment.status?.availableReplicas ?? 0 + if (updated >= desired && available >= desired) { + d.info('Sealed-secrets controller restarted successfully') + return + } + } catch { + // Ignore transient read errors during rollout + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)) } + d.warn('Rollout status check timed out') } /** From 16f6b279352deff273deedbd372668e31008b806 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:47:23 +0100 Subject: [PATCH 12/71] fix: sealed secret tests --- src/common/sealed-secrets.ts | 61 +++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 271281b76a..8bbe855eca 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -160,7 +160,10 @@ export const getExistingSealedSecretsCert = async (deps = { k8s, terminal }): Pr d.info('No existing sealed-secrets-key found') return undefined } - throw error + // When the cluster is unreachable (e.g., CI environment without a real cluster), + // treat it as no existing cert found and let bootstrap generate a new key pair. + d.info(`Could not reach cluster to check for existing cert: ${error instanceof Error ? error.message : error}`) + return undefined } } @@ -176,36 +179,44 @@ export const createSealedSecretsKeySecret = async ( ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - await ensureNamespaceExists('sealed-secrets') + try { + await ensureNamespaceExists('sealed-secrets') - // Check if secret already exists - const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') - if (existing) { - d.info('sealed-secrets-key already exists, skipping creation') - return - } + // Check if secret already exists + const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') + if (existing) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } - d.info('Creating sealed-secrets TLS secret') + d.info('Creating sealed-secrets TLS secret') - await k8s.core().createNamespacedSecret({ - namespace: 'sealed-secrets', - body: { - metadata: { - name: 'sealed-secrets-key', - namespace: 'sealed-secrets', - labels: { - 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + await k8s.core().createNamespacedSecret({ + namespace: 'sealed-secrets', + body: { + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { + 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + }, + }, + type: 'kubernetes.io/tls', + data: { + 'tls.crt': b64enc(certificate), + 'tls.key': b64enc(privateKey), }, }, - type: 'kubernetes.io/tls', - data: { - 'tls.crt': b64enc(certificate), - 'tls.key': b64enc(privateKey), - }, - }, - }) + }) - d.info('Created sealed-secrets TLS secret with key label') + d.info('Created sealed-secrets TLS secret with key label') + } catch (error) { + // When the cluster is unreachable (e.g., CI/bootstrap without a real cluster), + // skip secret creation. The secret will be created during install when the cluster is available. + d.info( + `Could not create sealed-secrets-key in cluster (will be created during install): ${error instanceof Error ? error.message : error}`, + ) + } } /** From 9f62106fdc92c83abe0f71b06b9ad6012284e6a4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:03:03 +0100 Subject: [PATCH 13/71] feat: remove init and prepare endpoints --- src/server.ts | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/src/server.ts b/src/server.ts index d989d8fc1e..65f24d1106 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,12 +1,8 @@ import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser' import express, { Request, Response } from 'express' -import { copyFile } from 'fs/promises' import { Server } from 'http' -import { bootstrapSops } from 'src/cmd/bootstrap' -import { decrypt, encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { hfValues } from './common/hf' -import { setValuesFile, unsetValuesFile } from './common/repo' import { loadYaml, rootDir } from './common/utils' import { objectToYaml } from './common/values' @@ -24,47 +20,8 @@ app.get('/', async (req: Request, res: Response): Promise => { type QueryParams = { envDir: string - files?: string[] } -app.get('/init', async (req: Request, res: Response): Promise => { - const { envDir } = req.query as QueryParams - try { - d.log('Request to initialize values repo on', envDir) - await decrypt(envDir) - res.status(200).send('ok') - } catch (error) { - d.error(error) - res.status(500).send(`${error}`) - } -}) - -app.get('/prepare', async (req: Request, res: Response): Promise => { - const { envDir, files } = req.query as QueryParams - try { - d.log('Request to prepare values repo on', envDir) - const file = '.editorconfig' - await copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - await bootstrapSops(envDir) - await setValuesFile(envDir) - // Encrypt ensures that a brand new secret file is encrypted in place - await encrypt(envDir, ...(files ?? [])) - // Decrypt ensures that a brand new encrypted secret file is decrypted to the .dec file - await decrypt(envDir, ...(files ?? [])) - res.status(200).send('ok') - } catch (error) { - const err = `${error}` - let status = 500 - d.error(`Request to prepare values went wrong: ${err}`) - if (err.includes('Values validation FAILED')) { - status = 422 - } - res.status(status).send(err) - } finally { - await unsetValuesFile(envDir) - } -}) - function parseBoolean(string: any, defaultValue = false): boolean { return string === 'true' ? true : string === 'false' ? false : defaultValue } From ea466f9ea016d9547518e3ad8ee4f8458eacc196 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:45:17 +0100 Subject: [PATCH 14/71] fix: harbor secrets --- values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl | 2 +- values/harbor/harbor-raw.gotmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 72ad5eb791..0eaf85f5a3 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -35,7 +35,7 @@ resources: data: - secretKey: harborAdminPassword remoteRef: - key: harbor-secrets + key: otomi-platform-secrets property: adminPassword - secretKey: keycloakClientSecret remoteRef: diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index feb9862641..fde1bcc6c9 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -52,7 +52,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: harbor-secrets + key: otomi-platform-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret From db046ba7c37ecbe9c71332aa3e467ec82a98cf7a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:26:53 +0100 Subject: [PATCH 15/71] feat: update tools image and remove /apl/schema endpoint --- .github/workflows/otomi-tools-build-push.yaml | 1 + charts/otomi-api/values.yaml | 7 -- src/server.ts | 9 --- tools/Dockerfile | 65 +------------------ versions.yaml | 4 +- 5 files changed, 4 insertions(+), 82 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3e5a7e0677 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index f4ddb4695e..280ebff005 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -112,10 +112,3 @@ tools: # requests: # cpu: 100m # memory: 100Mi - - secrets: - SOPS_AGE_KEY: '' - GCLOUD_SERVICE_KEY: '' - AZURE_TENANT_ID: '' - AZURE_CLIENT_ID: '' - AZURE_CLIENT_SECRET: '' diff --git a/src/server.ts b/src/server.ts index 65f24d1106..347cb12e5b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,9 +1,7 @@ -import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser' import express, { Request, Response } from 'express' import { Server } from 'http' import { terminal } from 'src/common/debug' import { hfValues } from './common/hf' -import { loadYaml, rootDir } from './common/utils' import { objectToYaml } from './common/values' const d = terminal('server') @@ -44,13 +42,6 @@ app.get('/otomi/values', async (req: Request, res: Response): Promise => { } }) -app.get('/apl/schema', async (req: Request, res: Response): Promise => { - const schema = await loadYaml(`${rootDir}/values-schema.yaml`) - const derefSchema = await $RefParser.dereference(schema as JSONSchema) - res.setHeader('Content-type', 'application/json') - res.status(200).send(derefSchema) -}) - export const startServer = (): void => { server = app .listen(17771, '127.0.0.1', () => { diff --git a/tools/Dockerfile b/tools/Dockerfile index af6f1b4836..b957ee09fc 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -3,30 +3,14 @@ FROM ubuntu:24.04 AS builder ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH -# https://github.com/kubernetes/kubernetes/releases -ARG KUBECTL_VERSION=1.34.2 # https://github.com/helm/helm/tags ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 -# https://github.com/jkroepke/helm-secrets/releases -ARG HELM_SECRETS_VERSION=4.7.4 -# https://github.com/mozilla/sops/releases -ARG SOPS_VERSION=3.11.0 -# https://github.com/FiloSottile/age/releases -ARG AGE_VERSION=1.2.1 -# https://github.com/noqcks/gucci/releases -ARG GUCCI_VERSION=1.9.0 -# https://github.com/yannh/kubeconform/releases/ -ARG KUBECONFORM_VERSION=0.7.0 # https://github.com/helmfile/helmfile/releases ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ ARG NODE_VERSION=24 -# https://github.com/cloudnative-pg/cloudnative-pg/releases -ARG CNPG_VERSION=1.27.1 -# https://github.com/jqlang/jq/releases/ -ARG JQ_VERSION=1.8.1 ARG HELM_FILE_NAME=helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz WORKDIR / @@ -34,26 +18,12 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ - coreutils \ - apache2-utils \ - apt-transport-https \ ca-certificates \ git \ - locales \ - rsync && \ + locales && \ rm -rf /var/lib/apt/lists/* && \ locale-gen en_US.UTF-8 -# jq -RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${TARGETARCH}" && \ - curl -L "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt" | grep "jq-linux-${TARGETARCH}" > sha256sum.txt && \ - echo sha256sum --check sha256sum.txt && \ - mv jq-linux-${TARGETARCH} /usr/bin/jq && \ - chmod +x /usr/bin/jq - -# yq -COPY --from=mikefarah/yq:4 /usr/bin/yq /usr/bin/yq - RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -63,48 +33,15 @@ RUN mkdir $APP_HOME WORKDIR $APP_HOME ENV PATH $PATH:$APP_HOME -# kubectl -RUN curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl" && \ - curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl.sha256" && \ - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check && \ - chmod +x kubectl - -# cnpg kubectl plugin -RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo "${TARGETARCH}"; fi) && \ - curl -LO "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${CNPG_VERSION}/kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz" && \ - tar -zxvf kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz && \ - chmod +x kubectl-cnpg && \ - rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz - -# sops -ADD https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64 sops -RUN chmod +x sops - -# age -ADD https://github.com/FiloSottile/age/releases/download/v${AGE_VERSION}/age-v${AGE_VERSION}-linux-amd64.tar.gz age.tar.gz -RUN tar -zxvf age.tar.gz age/age age/age-keygen --strip-components=1 && \ - chmod +x age age-keygen && \ - rm -rf age.tar.gz - # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_DIFF_VERSION} -RUN echo "exec \$*" > /usr/bin/sudo && chmod +x /usr/bin/sudo -RUN helm plugin install https://github.com/jkroepke/helm-secrets --version ${HELM_SECRETS_VERSION} # helmfile ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp RUN tar -zxvf /tmp/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz -C /tmp && mv /tmp/helmfile helmfile -# gucci -ADD https://github.com/noqcks/gucci/releases/download/v${GUCCI_VERSION}/gucci-v${GUCCI_VERSION}-linux-${TARGETARCH} gucci -RUN chmod +x gucci - -# kubeconform -ADD https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${TARGETARCH}.tar.gz /tmp -RUN tar -zxvf /tmp/kubeconform-linux-${TARGETARCH}.tar.gz -C /tmp && mv /tmp/kubeconform kubeconform - # node # https://github.com/nodesource/distributions RUN set -uex && \ diff --git a/versions.yaml b/versions.yaml index cba4e40597..9ec41ef7f3 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ -api: main +api: APL-523 console: main consoleLogin: main tasks: APL-1476-1 -tools: main +tools: APL-523 From d48c3bb96c9f9ea5c763bccf12221a50892d15fe Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:37:13 +0100 Subject: [PATCH 16/71] fix: versions --- .github/workflows/otomi-tools-build-push.yaml | 1 - versions.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3e5a7e0677..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode diff --git a/versions.yaml b/versions.yaml index 9ec41ef7f3..29fc9f62a2 100644 --- a/versions.yaml +++ b/versions.yaml @@ -2,4 +2,4 @@ api: APL-523 console: main consoleLogin: main tasks: APL-1476-1 -tools: APL-523 +tools: 2.10.7 From 286faf35c75778cc0fe095372a0341050d70e927 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 12:38:57 +0100 Subject: [PATCH 17/71] test: tools image --- .github/workflows/main.yml | 2 +- Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index feb4919bc9..02a3db4543 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,7 @@ jobs: if: always() && contains(needs.release.result, 'success') && !github.event.act runs-on: ubuntu-22.04 container: - image: linode/apl-tools:v2.10.6 + image: linode/apl-tools:v2.10.7 options: --user 0 # See https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user steps: - name: Checkout diff --git a/Dockerfile b/Dockerfile index caa68fc085..58a4f88252 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM linode/apl-tools:v2.10.6 AS ci +FROM linode/apl-tools:v2.10.7 AS ci ENV APP_HOME=/home/app/stack @@ -36,7 +36,7 @@ FROM ci AS clean # below command removes the packages specified in devDependencies and set NODE_ENV to production RUN npm prune --production -FROM linode/apl-tools:v2.10.6 AS prod +FROM linode/apl-tools:v2.10.7 AS prod ARG APPS_REVISION='' ENV APP_HOME=/home/app/stack ENV ENV_DIR=/home/app/stack/env From 41a30f919ed8ffb5a7062c94717239e4f784d497 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 17:54:25 +0100 Subject: [PATCH 18/71] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 ++++++- tools/Dockerfile | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3d38c4bc3f 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode @@ -70,7 +71,11 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - echo "No need to bump the version. Will skip next steps." + + # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored + NEW_VERSION="v2.10.7" + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + echo "Overriding NEW_VERSION=${NEW_VERSION}" - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/tools/Dockerfile b/tools/Dockerfile index b957ee09fc..559b5f30a9 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -7,6 +7,10 @@ ARG TARGETARCH ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 +# https://github.com/noqcks/gucci/releases +ARG GUCCI_VERSION=1.9.0 +# https://github.com/yannh/kubeconform/releases/ +ARG KUBECONFORM_VERSION=0.7.0 # https://github.com/helmfile/helmfile/releases ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ @@ -18,6 +22,7 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ + apache2-utils \ ca-certificates \ git \ locales && \ @@ -42,6 +47,14 @@ RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_ ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp RUN tar -zxvf /tmp/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz -C /tmp && mv /tmp/helmfile helmfile +# gucci +ADD https://github.com/noqcks/gucci/releases/download/v${GUCCI_VERSION}/gucci-v${GUCCI_VERSION}-linux-${TARGETARCH} gucci +RUN chmod +x gucci + +# kubeconform +ADD https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${TARGETARCH}.tar.gz /tmp +RUN tar -zxvf /tmp/kubeconform-linux-${TARGETARCH}.tar.gz -C /tmp && mv /tmp/kubeconform kubeconform + # node # https://github.com/nodesource/distributions RUN set -uex && \ From 1644634f29a67cb31c326602ca419cb8af8b7904 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 18:30:18 +0100 Subject: [PATCH 19/71] feat: remove kms from bootstrap files --- tests/bootstrap/input-local-dev.yaml | 3 --- tests/bootstrap/input.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/bootstrap/input-local-dev.yaml b/tests/bootstrap/input-local-dev.yaml index ccaaafd9eb..1f7cdfd0e5 100644 --- a/tests/bootstrap/input-local-dev.yaml +++ b/tests/bootstrap/input-local-dev.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: true diff --git a/tests/bootstrap/input.yaml b/tests/bootstrap/input.yaml index 0cb49d23af..64d426b912 100644 --- a/tests/bootstrap/input.yaml +++ b/tests/bootstrap/input.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: false From f3755a640d04fb5bc2163b4828e6053cb19c97f7 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 18:30:59 +0100 Subject: [PATCH 20/71] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3d38c4bc3f..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode @@ -71,11 +70,7 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - - # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored - NEW_VERSION="v2.10.7" - echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV - echo "Overriding NEW_VERSION=${NEW_VERSION}" + echo "No need to bump the version. Will skip next steps." - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} From 18d84accf4720f02479863c3bf8956184d7dc632 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 20:19:21 +0100 Subject: [PATCH 21/71] feat: remove kms and sops related code --- .github/workflows/integration.yml | 13 - src/cmd/bootstrap.test.ts | 624 ++++++++++++------------------ src/cmd/bootstrap.ts | 170 +------- src/common/bootstrap.ts | 3 - src/common/envalid.ts | 1 - src/operator/installer.test.ts | 19 +- src/operator/installer.ts | 24 +- 7 files changed, 247 insertions(+), 607 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b273c7964f..d3b5bf0f7a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,10 +22,6 @@ on: description: 'Select Domain Zone' type: string default: DNS-Integration - kms: - description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' - type: string - default: age certificate: description: 'Select certificate issuer' type: string @@ -85,13 +81,6 @@ on: - Zone-2 - Random - DNS-Integration - kms: - type: choice - description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? - options: - - age - - no_kms - default: age certificate: type: choice description: Select certificate issuer @@ -142,7 +131,6 @@ jobs: echo 'install_profile: ${{ inputs.install_profile }}' echo 'lke_tier: ${{ inputs.lke_tier }}' echo 'kubernetes_version: ${{ inputs.kubernetes_version }}' - echo 'kms: ${{ inputs.kms }}' echo 'domain_zone: ${{ inputs.domain_zone }}' echo 'certificate: ${{ inputs.certificate }}' echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' @@ -314,7 +302,6 @@ jobs: additional_args="" [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml - [[ '${{ inputs.kms }}' == 'age' ]] && additional_args+=" --set kms.sops.provider=age" [[ '${{ inputs.is_pre_installed }}' == 'true' ]] && additional_args+=" --set otomi.isPreInstalled=true" if [[ '${{ inputs.disableORCS }}' == 'true' ]]; then additional_args+=" --set otomi.useORCS=false" diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 0a1b004fc7..44e2684cda 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -3,10 +3,8 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { bootstrap, - bootstrapSops, copyBasicFiles, createCustomCA, - getKmsValues, getStoredClusterSecrets, handleFileEntry, processValues, @@ -38,23 +36,13 @@ describe('Bootstrapping values', () => { quiet: jest.fn(), }), }), - bootstrapSops: jest.fn(), bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), handleFileEntry: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - existsSync: jest.fn(), - genSops: jest.fn(), - getDeploymentState: jest.fn().mockReturnValue({}), - getImageTagFromValues: jest.fn(), getK8sSecret: jest.fn(), - hfValues: jest.fn(), - isCli: true, migrate: jest.fn(), - nothrow: jest.fn(), pathExists: jest.fn(), processValues: jest.fn(), terminal, @@ -63,7 +51,6 @@ describe('Bootstrapping values', () => { }) it('should call relevant sub routines', async () => { deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) - deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() @@ -71,7 +58,6 @@ describe('Bootstrapping values', () => { it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) }) it('should get stored cluster secrets if those exist', async () => { deps.getK8sSecret.mockReturnValue({ 'otomi-generated-passwords': secrets }) @@ -84,402 +70,274 @@ describe('Bootstrapping values', () => { expect(res).toEqual(undefined) }) - describe('getKmsValues', () => { - let kmsValuesDeps: any - const ageKeys = { publicKey: 'agePublicKey', privateKey: 'agePrivateKey' } - const values = { someKey: 'someValue' } - beforeEach(() => { - kmsValuesDeps = { - generateAgeKeys: jest.fn().mockResolvedValue(ageKeys), - hfValues: jest.fn(), - } - }) - it('should not get kms values if those do not exist', async () => { - kmsValuesDeps.hfValues.mockReturnValue(values) - const res = await getKmsValues(kmsValuesDeps) - expect(res).toBeUndefined() + describe('Copying basic files', () => { + const deps = { + copy: jest.fn(), + copyFile: jest.fn(), + copySchema: jest.fn(), + mkdir: jest.fn(), + pathExists: jest.fn(), + terminal, + } + it('should not throw any exception', async () => { + const res = await copyBasicFiles(deps) + expect(res).toBe(undefined) }) - it('should get kms values if those exist', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'azure', - }, - }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'azure', - }, + }) + describe('Creating folders and files for workload', () => { + const values = { + values: { + image: { + repository: 'linode/apl-nodejs-helloworld', + tag: 'v1.5.1', }, - }) + }, + } + const workload = { + files: { + 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), + }, + } + const deps = { + loadYaml: jest.fn().mockReturnValue(workload), + mkdir: jest.fn(), + terminal, + writeFile: jest.fn(), + } + it('should create folders and files based on file entry in yaml', async () => { + await handleFileEntry(deps) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/teams/workloads/demo/values.yaml', JSON.stringify(values)) }) - it('should generate and return new age keys if provider is age and keys are missing', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'age', - }, + }) + describe('Checking for a custom CA', () => { + const deps = { + pki: { + rsa: { + generateKeyPair: jest.fn().mockReturnValue({ + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + }), }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, + createCertificate: jest.fn().mockReturnValue({ + publicKey: {}, + serialNumber: '01', + validity: {}, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + }), + certificateToPem: jest.fn(), + privateKeyToPem: jest.fn(), + } as unknown as typeof pki, + writeValues: jest.fn(), + terminal, + } + deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') + deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') + it('should create a new key pair when none exist', () => { + const res = createCustomCA(deps) + expect(res).toMatchObject({ + apps: { + 'cert-manager': { + customRootCA: 'certpem', + customRootCAKey: 'keypem', }, }, }) }) - describe('Copying basic files', () => { - const deps = { - copy: jest.fn(), - copyFile: jest.fn(), - copySchema: jest.fn(), - mkdir: jest.fn(), - pathExists: jest.fn(), + }) + describe('processing values', () => { + const generatedSecrets = { gen: 'x' } + const generatedPassword = 'generated-password' + const usersWithPasswords = [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: generatedPassword }, + ] + // Pre-processed users (as stored in allSecrets for sealed secret generation) + const processedUsers = usersWithPasswords.map((u: any) => ({ + email: u.email, + firstName: u.firstName, + lastName: u.lastName, + initialPassword: u.initialPassword, + groups: [ + ...(u.isPlatformAdmin ? ['platform-admin'] : []), + ...(u.isTeamAdmin ? ['team-admin'] : []), + ...(u.teams || []).map((t: string) => `team-${t}`), + ], + })) + const ca = { a: 'cert' } + const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) + const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) + const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) + let deps + beforeEach(() => { + deps = { + createCustomCA: jest.fn().mockReturnValue(ca), + createK8sSecret: jest.fn(), + generateSecrets: jest.fn().mockReturnValue(generatedSecrets), + getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), + loadYaml: jest.fn(), terminal, + writeValues: jest.fn(), + getUsers: jest.fn().mockReturnValue(usersWithPasswords), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), + stripAllSecrets: jest.fn().mockImplementation((v) => v), } - it('should not throw any exception', async () => { - const res = await copyBasicFiles(deps) - expect(res).toBe(undefined) - }) }) - describe('Creating folders and files for workload', () => { - const values = { - values: { - image: { - repository: 'linode/apl-nodejs-helloworld', - tag: 'v1.5.1', - }, - }, - } - const workload = { - files: { - 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), - }, - } - const deps = { - loadYaml: jest.fn().mockReturnValue(workload), - mkdir: jest.fn(), - terminal, - writeFile: jest.fn(), - } - it('should create folders and files based on file entry in yaml', async () => { - await handleFileEntry(deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) - expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/teams/workloads/demo/values.yaml', - JSON.stringify(values), - ) + describe('Creating CA', () => { + it('should ask to create a CA if issuer is custom-ca', async () => { + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalledTimes(1) }) }) - describe('Generating sops related files', () => { - const settings = { - kms: { - sops: { - provider: 'aws', - aws: { - keys: 'key1,key2', - }, - }, - }, - } - const deps = { - copyFile: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - gucci: jest.fn().mockReturnValue('ok'), - hfValues: jest.fn(), - loadYaml: jest.fn().mockReturnValue(settings), - pathExists: jest.fn(), - readFile: jest.fn(), - getKmsSettings: jest.fn(), - terminal, - writeFile: jest.fn(), - createUpdateGenericSecret: jest.fn(), - } - it('should create files on first run and en/de-crypt', async () => { - deps.pathExists.mockReturnValue(false) - deps.getKmsSettings.mockReturnValue({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, - }, - }, - }) - - await bootstrapSops(undefined, deps) - expect(deps.encrypt).toHaveBeenCalled() - expect(deps.decrypt).toHaveBeenCalled() + describe('processing app values', () => { + it('should generate secrets by taking values and previously generated secrets as input', async () => { + deps.loadYaml.mockReturnValue(values) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - it('should just create files on next runs', async () => { - deps.pathExists.mockReturnValue(true) - deps.hfValues.mockReturnValue(settings) - deps.decrypt = jest.fn() - deps.encrypt = jest.fn() - const res = await bootstrapSops(undefined, deps) - expect(res).toBe(undefined) - expect(deps.encrypt).not.toHaveBeenCalled() - expect(deps.decrypt).not.toHaveBeenCalled() + it('should overwrite a stored secret with one that was provided in values', async () => { + const newSecret = { secret: 'new' } + const valuesWithSecrets = merge(cloneDeep(values), newSecret) + const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) + deps.loadYaml.mockReturnValue(valuesWithSecrets) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(allSecrets) + await processValues(deps) + const expected = { ...allSecrets, users: processedUsers } + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - }) - describe('Checking for a custom CA', () => { - const deps = { - pki: { - rsa: { - generateKeyPair: jest.fn().mockReturnValue({ - publicKey: { n: {}, e: {} }, - privateKey: { d: {}, p: {}, q: {} }, - }), - }, - createCertificate: jest.fn().mockReturnValue({ - publicKey: {}, - serialNumber: '01', - validity: {}, - sign: jest.fn(), - setSubject: jest.fn(), - setIssuer: jest.fn(), - setExtensions: jest.fn(), - }), - certificateToPem: jest.fn(), - privateKeyToPem: jest.fn(), - } as unknown as typeof pki, - writeValues: jest.fn(), - terminal, - } - deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') - deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') - it('should create a new key pair when none exist', () => { - const res = createCustomCA(deps) - expect(res).toMatchObject({ - apps: { - 'cert-manager': { - customRootCA: 'certpem', - customRootCAKey: 'keypem', - }, - }, - }) - }) - }) - describe('processing values', () => { - const generatedSecrets = { gen: 'x' } - const generatedPassword = 'generated-password' - const usersWithPasswords = [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: generatedPassword }, - ] - // Pre-processed users (as stored in allSecrets for sealed secret generation) - const processedUsers = usersWithPasswords.map((u: any) => ({ - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - initialPassword: u.initialPassword, - groups: [ - ...(u.isPlatformAdmin ? ['platform-admin'] : []), - ...(u.isTeamAdmin ? ['team-admin'] : []), - ...(u.teams || []).map((t: string) => `team-${t}`), - ], - })) - const ca = { a: 'cert' } - const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) - const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) - const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) - let deps - beforeEach(() => { - deps = { - createCustomCA: jest.fn().mockReturnValue(ca), - createK8sSecret: jest.fn(), - decrypt: jest.fn(), - existsSync: jest.fn(), - generateSecrets: jest.fn().mockReturnValue(generatedSecrets), - getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), - getKmsValues: jest.fn().mockReturnValue({}), - hfValues: jest.fn().mockReturnValue(values), - loadYaml: jest.fn(), - terminal, - validateValues: jest.fn().mockReturnValue(true), - writeValues: jest.fn(), - getUsers: jest.fn().mockReturnValue(usersWithPasswords), - generatePassword: jest.fn().mockReturnValue(generatedPassword), - addInitialPasswords: jest.fn().mockReturnValue(usersWithPasswords), - addPlatformAdmin: jest.fn().mockReturnValue(usersWithPasswords), - pathExists: jest.fn().mockReturnValue(true), - getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), - stripAllSecrets: jest.fn().mockImplementation((v) => v), - } + it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { + deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalled() }) - describe('Creating CA', () => { - it('should ask to create a CA if issuer is custom-ca', async () => { - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalledTimes(1) + it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { + deps.loadYaml.mockReturnValue({ + apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, }) + await processValues(deps) + expect(deps.createCustomCA).not.toHaveBeenCalled() }) - describe('processing app values', () => { - it('should not retrieve values from env dir', async () => { - await processValues(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) - }) - it('should generate secrets by taking values and previously generated secrets as input', async () => { - deps.loadYaml.mockReturnValue(values) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) - }) - it('should overwrite a stored secret with one that was provided in values', async () => { - const newSecret = { secret: 'new' } - const valuesWithSecrets = merge(cloneDeep(values), newSecret) - const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) - deps.loadYaml.mockReturnValue(valuesWithSecrets) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(allSecrets) - await processValues(deps) - const expected = { ...allSecrets, users: processedUsers } - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) + it('should only store secrets', async () => { + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + await processValues(deps) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...mergedSecretsWithGenAndCa, + users: processedUsers, }) - it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { - deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalled() - }) - it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { - deps.loadYaml.mockReturnValue({ - apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, - }) - await processValues(deps) - expect(deps.createCustomCA).not.toHaveBeenCalled() - }) - it('should only store secrets', async () => { - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue(ca) - await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { - ...mergedSecretsWithGenAndCa, - users: processedUsers, - }) + }) + it('should not overwrite stored secrets', async () => { + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue({}) + deps.generateSecrets.mockReturnValue(generatedSecrets) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...generatedSecrets, + users: processedUsers, }) - it('should not overwrite stored secrets', async () => { - deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue({}) - deps.generateSecrets.mockReturnValue(generatedSecrets) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { - ...generatedSecrets, - users: processedUsers, - }) + }) + it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { + deps.loadYaml.mockReturnValue({ + cluster: { name: 'bla', provider: 'dida' }, }) - it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { - deps.loadYaml.mockReturnValue({ - cluster: { name: 'bla', provider: 'dida' }, - }) - deps.getStoredClusterSecrets.mockReturnValue({ - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) - deps.generateSecrets.mockReturnValue({ gen: 'x' }) - deps.createCustomCA.mockReturnValue(ca) - const res = await processValues(deps) - // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) - // processedUsers adds groups:[] to each user via element-wise lodash merge - expect(deps.writeValues).toHaveBeenNthCalledWith(1, { - a: 'cert', - gen: 'x', - cluster: { name: 'bla', provider: 'dida' }, - users: [ - { id: 'user1', initialPassword: 'existing-password', groups: [] }, - { id: 'user2', initialPassword: 'generated-password', groups: [] }, - ], - }) - expect(res.originalInput).toEqual({ - cluster: { name: 'bla', provider: 'dida' }, - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) + deps.getStoredClusterSecrets.mockReturnValue({ + users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) - it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { - // mergedForDisk = merge(originalInput, allSecrets, { users }) - // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers - const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { - users: processedUsers, - }) - const expectedDiskValues = merge( - cloneDeep(secrets), - cloneDeep(values), - cloneDeep(allSecretsExpected), - cloneDeep({ users: usersWithPasswords }), - ) - deps.loadYaml.mockReturnValue({ ...values, users }) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.getUsers.mockReturnValue(usersWithPasswords) - await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + deps.generateSecrets.mockReturnValue({ gen: 'x' }) + deps.createCustomCA.mockReturnValue(ca) + const res = await processValues(deps) + // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) + // processedUsers adds groups:[] to each user via element-wise lodash merge + expect(deps.writeValues).toHaveBeenNthCalledWith(1, { + a: 'cert', + gen: 'x', + cluster: { name: 'bla', provider: 'dida' }, + users: [ + { id: 'user1', initialPassword: 'existing-password', groups: [] }, + { id: 'user2', initialPassword: 'generated-password', groups: [] }, + ], }) - it('should call stripAllSecrets before writing values to disk', async () => { - deps.loadYaml.mockReturnValue(values) - deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) - await processValues(deps) - expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) - expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + expect(res.originalInput).toEqual({ + cluster: { name: 'bla', provider: 'dida' }, + users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) - it('should still return full allSecrets for bootstrapSealedSecrets', async () => { - deps.loadYaml.mockReturnValue(values) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue(ca) - const result = await processValues(deps) - // allSecrets should contain full unstripped secrets including pre-processed users - expect(result.allSecrets).toEqual( - merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), - ) + }) + it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { + // mergedForDisk = merge(originalInput, allSecrets, { users }) + // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { + users: processedUsers, }) - it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { - // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) - const storedProcessedUsers = [ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ] - deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) - deps.generateSecrets.mockReturnValue({}) - deps.createCustomCA.mockReturnValue({}) - // getUsers returns the stored processed users (no isPlatformAdmin flag) - deps.getUsers.mockReturnValue(storedProcessedUsers) + const expectedDiskValues = merge( + cloneDeep(secrets), + cloneDeep(values), + cloneDeep(allSecretsExpected), + cloneDeep({ users: usersWithPasswords }), + ) + deps.loadYaml.mockReturnValue({ ...values, users }) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.getUsers.mockReturnValue(usersWithPasswords) + await processValues(deps) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + }) + it('should call stripAllSecrets before writing values to disk', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) + await processValues(deps) + expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + }) + it('should still return full allSecrets for bootstrapSealedSecrets', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + const result = await processValues(deps) + // allSecrets should contain full unstripped secrets including pre-processed users + expect(result.allSecrets).toEqual( + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + ) + }) + it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { + // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) + const storedProcessedUsers = [ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ] + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.generateSecrets.mockReturnValue({}) + deps.createCustomCA.mockReturnValue({}) + // getUsers returns the stored processed users (no isPlatformAdmin flag) + deps.getUsers.mockReturnValue(storedProcessedUsers) - const result = await processValues(deps) + const result = await processValues(deps) - // Groups should be preserved from existing data, not reset to [] - expect(result.allSecrets.users).toEqual([ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ]) - }) + // Groups should be preserved from existing data, not reset to [] + expect(result.allSecrets.users).toEqual([ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ]) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 99da59dac5..06842b9486 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'crypto' import { existsSync } from 'fs' -import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises' +import { copyFile, cp, mkdir, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' import { cloneDeep, get, merge, set } from 'lodash' import { pki } from 'node-forge' @@ -8,25 +8,14 @@ import path from 'path' import { bootstrapGit } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' import { DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' -import { decrypt, encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' -import { - createK8sSecret, - createUpdateGenericSecret, - getDeploymentState, - getK8sSecret, - k8s, - secretId, -} from 'src/common/k8s' -import { getKmsSettings } from 'src/common/repo' +import { createK8sSecret, getK8sSecret, secretId } from 'src/common/k8s' import { bootstrapSealedSecrets, stripAllSecrets } from 'src/common/sealed-secrets' import { ensureTeamGitOpsDirectories, getFilename, getSchemaSecretsPaths, - gucci, isCore, loadYaml, rootDir, @@ -36,111 +25,9 @@ import { BasicArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' import { $ } from 'zx' import { migrate } from './migrate' -import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -const kmsMap = { - aws: 'kms', - azure: 'azure_keyvault', - google: 'gcp_kms', - age: 'age', -} - -export const bootstrapSops = async ( - envDir = env.ENV_DIR, - deps = { - copyFile, - decrypt, - encrypt, - gucci, - loadYaml, - pathExists: existsSync, - getKmsSettings, - readFile, - terminal, - writeFile, - createUpdateGenericSecret, - }, -): Promise => { - const d = deps.terminal(`cmd:${cmdName}:genSops`) - const targetPath = `${envDir}/.sops.yaml` - const values = await deps.getKmsSettings(envDir) - - const provider: string | undefined = values?.kms?.sops?.provider - if (!provider) { - d.warn('No sops information given. Assuming no sops enc/decryption needed. Be careful!') - return - } - - const templatePath = `${rootDir}/tpl/.sops.yaml.gotmpl` - const kmsProvider = kmsMap[provider] as string - const kmsKeys = values.kms.sops[provider]?.keys as string - - const obj = { - provider: kmsProvider, - keys: kmsKeys, - } - - if (provider === 'age') { - const { publicKey } = values?.kms?.sops?.age ?? {} - let privateKey = values.kms?.sops?.age?.privateKey - if (privateKey?.startsWith('ENC')) { - privateKey = '' - } - obj.keys = publicKey - if (privateKey && !process.env.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(`${envDir}/.secrets`, `SOPS_AGE_KEY=${privateKey}`) - try { - await deps.createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: privateKey, - }) - } catch (e) { - d.warn('Failed to create or update apl-sops-secrets secret with SOPS_AGE_KEY, this might come later') - } - } - } - - const exists = deps.pathExists(targetPath) - d.log(`Creating sops file for provider ${provider}`) - const output = (await deps.gucci(templatePath, obj, true)) as string - await deps.writeFile(targetPath, output) - d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - - // prepare some credential files the first time and crypt some - if (!exists) { - if (isCli || env.OTOMI_DEV) { - // first time so we know we have values - const secretsFile = `${envDir}/.secrets` - d.log(`Creating secrets file: ${secretsFile}`) - if (provider === 'google') { - // and we also assume the correct values are given by using '!' (we want to err when not set) - const serviceKeyJson = JSON.parse(values.kms.sops!.google!.accountJson as string) - // and set it in env for later decryption - process.env.GCLOUD_SERVICE_KEY = values.kms.sops!.google!.accountJson - d.log('Creating gcp-key.json for vscode.') - await deps.writeFile(`${envDir}/gcp-key.json`, JSON.stringify(serviceKeyJson)) - d.log(`Creating credentials file: ${secretsFile}`) - await deps.writeFile(secretsFile, `GCLOUD_SERVICE_KEY=${JSON.stringify(JSON.stringify(serviceKeyJson))}`) - } else if (provider === 'aws') { - const v = values.kms.sops!.aws! - await deps.writeFile(secretsFile, `AWS_ACCESS_KEY_ID='${v.accessKey}'\nAWS_ACCESS_KEY_SECRET=${v.secretKey}`) - } else if (provider === 'azure') { - const v = values.kms.sops!.azure! - await deps.writeFile(secretsFile, `AZURE_CLIENT_ID='${v.clientId}'\nAZURE_CLIENT_SECRET=${v.clientSecret}`) - } else if (provider === 'age') { - const { privateKey } = values.kms.sops!.age! - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(secretsFile, `SOPS_AGE_KEY=${privateKey}`) - } - } - // now do a round of encryption and decryption to make sure we have all the files in place for later - await deps.encrypt(envDir) - await deps.decrypt(envDir) - } -} - export const copySchema = async (deps = { terminal, rootDir, env, isCore, loadYaml, copyFile }): Promise => { const d = deps.terminal(`cmd:${cmdName}:copySchema`) const { ENV_DIR } = env @@ -179,36 +66,6 @@ export const getStoredClusterSecrets = async ( return undefined } -export const generateAgeKeys = async (deps = { $, terminal }) => { - const d = deps.terminal(`cmd:${cmdName}:generateAgeKeys`) - try { - d.info('Generating age keys') - const result = await deps.$`age-keygen` - const { stdout } = result - const matchPublic = stdout?.match(/age[0-9a-z]+/) - const publicKey = matchPublic ? matchPublic[0] : '' - const matchPrivate = stdout?.match(/AGE-SECRET-KEY-[0-9A-Z]+/) - const privateKey = matchPrivate ? matchPrivate[0] : '' - const ageKeys = { publicKey, privateKey } - return ageKeys - } catch (error) { - d.log('Error generating age keys:', error) - throw error - } -} - -export const getKmsValues = async (values: Record, deps = { generateAgeKeys }) => { - const kms = values?.kms - if (!kms) return undefined - const provider = kms?.sops?.provider - if (!provider) return {} - if (provider !== 'age') return { kms } - const age = kms?.sops?.age - if (age?.publicKey && age?.privateKey) return { kms } - const ageKeys = await deps.generateAgeKeys() - return { kms: { sops: { provider: 'age', age: ageKeys } } } -} - export const addPlatformAdmin = (users: any[], domainSuffix: string) => { const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` const platformAdminExists = users.find((user) => user.email === defaultPlatformAdminEmail) @@ -295,20 +152,12 @@ export const processValues = async ( deps = { terminal, loadYaml, - decrypt, getStoredClusterSecrets, - getKmsValues, writeValues, - pathExists: existsSync, - hfValues, - validateValues, generateSecrets, createK8sSecret, createCustomCA, getUsers, - generatePassword, - addInitialPasswords, - addPlatformAdmin, getSchemaSecretsPaths, stripAllSecrets, }, @@ -329,15 +178,8 @@ export const processValues = async ( } else { caSecrets = deps.createCustomCA() } - // get any kms values & generate age keys if needed - const kmsValues = (await deps.getKmsValues(originalInput)) || {} // merge existing secrets over newly generated ones to keep them - const allSecrets = merge( - cloneDeep(caSecrets), - cloneDeep(storedSecrets), - cloneDeep(generatedSecrets), - cloneDeep(kmsValues), - ) + const allSecrets = merge(cloneDeep(caSecrets), cloneDeep(storedSecrets), cloneDeep(generatedSecrets)) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage @@ -458,17 +300,11 @@ export const createCustomCA = (deps = { terminal, pki, writeValues }): Record => { diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 7f1ce9d1d7..6e493353eb 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -90,8 +90,5 @@ export const bootstrapGit = async (inValues?: Record): Promise { jest.clearAllMocks() jest.useFakeTimers() - // Save original environment variables - process.env.SOPS_AGE_KEY = '' - mockCoreApi = {} ;(k8s.k8s.core as jest.Mock).mockReturnValue(mockCoreApi) @@ -336,22 +333,8 @@ describe('Installer', () => { }) describe('setEnvAndCreateSecrets', () => { - test('should use existing SOPS key from secret', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) - - await installer.setEnvAndCreateSecrets() - - expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') - expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') - }) - - test('should skip gracefully when SOPS key not found in secret (SealedSecrets in use)', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(null) - + test('should complete without errors', async () => { await installer.setEnvAndCreateSecrets() - - // Should not throw — SOPS is no longer required (replaced by SealedSecrets + ESO) - expect(process.env.SOPS_AGE_KEY).toBe('') }) }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index e55f28edfd..c4616cebd9 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,8 +1,7 @@ -import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -123,25 +122,6 @@ export class Installer { } public async setEnvAndCreateSecrets(): Promise { - this.d.debug('Retrieving or creating git credentials') - await this.setupSopsEnvironment() - } - - private async setupSopsEnvironment() { - try { - const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - - if (aplSopsSecret?.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY - this.d.debug('Using existing sops credentials from secret') - } else { - // SOPS is no longer used (replaced by SealedSecrets + ESO). - // Skip hfValues() call which requires the git repo that may not exist yet. - this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') - } - } catch (error) { - this.d.error('Failed to retrieve sops credentials:', getErrorMessage(error)) - throw error - } + this.d.debug('Environment setup complete (SOPS removed, using SealedSecrets + ESO)') } } From 8cd84f744c23cbeba6ad2821d9a88497c208506e Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 20:48:36 +0100 Subject: [PATCH 22/71] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 ++++- tools/Dockerfile | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3d38c4bc3f 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode @@ -70,7 +71,11 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - echo "No need to bump the version. Will skip next steps." + + # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored + NEW_VERSION="v2.10.7" + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + echo "Overriding NEW_VERSION=${NEW_VERSION}" - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/tools/Dockerfile b/tools/Dockerfile index 559b5f30a9..4f22ba1053 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -3,6 +3,8 @@ FROM ubuntu:24.04 AS builder ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH +# https://github.com/kubernetes/kubernetes/releases +ARG KUBECTL_VERSION=1.34.2 # https://github.com/helm/helm/tags ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases @@ -15,6 +17,10 @@ ARG KUBECONFORM_VERSION=0.7.0 ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ ARG NODE_VERSION=24 +# https://github.com/cloudnative-pg/cloudnative-pg/releases +ARG CNPG_VERSION=1.27.1 +# https://github.com/jqlang/jq/releases/ +ARG JQ_VERSION=1.8.1 ARG HELM_FILE_NAME=helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz WORKDIR / @@ -25,10 +31,18 @@ RUN apt-get update && apt-get install -y \ apache2-utils \ ca-certificates \ git \ - locales && \ + locales \ + rsync && \ rm -rf /var/lib/apt/lists/* && \ locale-gen en_US.UTF-8 +# jq +RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${TARGETARCH}" && \ + curl -L "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt" | grep "jq-linux-${TARGETARCH}" > sha256sum.txt && \ + echo sha256sum --check sha256sum.txt && \ + mv jq-linux-${TARGETARCH} /usr/bin/jq && \ + chmod +x /usr/bin/jq + RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -38,6 +52,19 @@ RUN mkdir $APP_HOME WORKDIR $APP_HOME ENV PATH $PATH:$APP_HOME +# kubectl +RUN curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl" && \ + curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl.sha256" && \ + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check && \ + chmod +x kubectl + +# cnpg kubectl plugin +RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo "${TARGETARCH}"; fi) && \ + curl -LO "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${CNPG_VERSION}/kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz" && \ + tar -zxvf kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz && \ + chmod +x kubectl-cnpg && \ + rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz + # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* From 13f73c41bb8bd2cf1aaf733bc02db1f5dd9d983f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 12:13:06 +0100 Subject: [PATCH 23/71] feat: update user management --- src/common/sealed-secrets.test.ts | 3 + src/common/sealed-secrets.ts | 174 +++++++++++++++++++++++++++++- src/operator/apl-operator.test.ts | 3 + src/operator/apl-operator.ts | 7 +- 4 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 7dbd238273..01d610f144 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -426,6 +426,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), } @@ -457,6 +458,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), createSealedSecretManifest: jest.fn().mockResolvedValue({}), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn(), } @@ -488,6 +490,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), createSealedSecretManifest: jest.fn(), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn(), } diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 8bbe855eca..a454c2f7ca 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -631,6 +631,153 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi * Orchestrator: bootstrap sealed secrets for the platform. * Replaces bootstrapSops(). */ +/** + * Aggregate individual user K8s secrets from apl-users namespace into a single + * users-secrets Secret in apl-secrets namespace (usersJson key). + * This feeds the Keycloak Helm chart configuration. + * Called during the operator reconcile loop. + */ +export const aggregateUserSecrets = async (deps = { terminal, k8s, b64enc }): Promise => { + const d = deps.terminal(`common:${cmdName}:aggregateUserSecrets`) + const namespace = 'apl-users' + const targetNamespace = 'apl-secrets' + const targetSecretName = 'users-secrets' + + try { + const res: any = await deps.k8s.core().listNamespacedSecret({ namespace }) + const users: any[] = [] + + for (const item of res.items || []) { + if (item.type !== 'Opaque') continue + if (!item.data?.email) continue + + const decoded: Record = {} + for (const [key, value] of Object.entries(item.data as Record)) { + decoded[key] = Buffer.from(value, 'base64').toString('utf-8') + } + + // Build user in keycloak-operator format (with groups) + const groups: string[] = [] + if (decoded.isPlatformAdmin === 'true') groups.push('platform-admin') + if (decoded.isTeamAdmin === 'true') groups.push('team-admin') + const teams = decoded.teams ? JSON.parse(decoded.teams) : [] + for (const team of teams) groups.push(`team-${team}`) + + users.push({ + email: decoded.email, + firstName: decoded.firstName || '', + lastName: decoded.lastName || '', + initialPassword: decoded.initialPassword || '', + groups, + }) + } + + if (users.length === 0) { + d.info('No user secrets found in apl-users namespace, skipping aggregation') + return + } + + const usersJson = JSON.stringify(users) + + // Create or update the users-secrets Secret in apl-secrets + await ensureNamespaceExists(targetNamespace) + + try { + await deps.k8s.core().readNamespacedSecret({ name: targetSecretName, namespace: targetNamespace }) + // Secret exists, patch it + await deps.k8s.core().patchNamespacedSecret( + { + name: targetSecretName, + namespace: targetNamespace, + body: { + data: { usersJson: deps.b64enc(usersJson) }, + }, + }, + setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), + ) + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + // Secret doesn't exist, create it + await deps.k8s.core().createNamespacedSecret({ + namespace: targetNamespace, + body: { + metadata: { + name: targetSecretName, + namespace: targetNamespace, + }, + type: 'Opaque', + data: { usersJson: deps.b64enc(usersJson) }, + }, + }) + } else { + throw error + } + } + + d.info(`Aggregated ${users.length} user(s) into ${targetNamespace}/${targetSecretName}`) + } catch (error) { + d.warn(`Failed to aggregate user secrets: ${error instanceof Error ? error.message : error}`) + } +} + +/** + * Create individual SealedSecret manifests for each user in the apl-users namespace. + * Each user gets their own SealedSecret with all fields encrypted. + */ +export const createUserSealedSecretManifests = async ( + users: any[], + pem: string, + deps = { encryptSecretItem, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createUserSealedSecretManifests`) + const namespace = 'apl-users' + const manifests: SealedSecretManifest[] = [] + + for (const user of users) { + const userId = user.name || user.id + if (!userId) { + d.warn('Skipping user without id/name') + continue + } + + const data: Record = { + email: user.email || '', + firstName: user.firstName || '', + lastName: user.lastName || '', + initialPassword: user.initialPassword || '', + isPlatformAdmin: String(user.isPlatformAdmin ?? false), + isTeamAdmin: String(user.isTeamAdmin ?? false), + teams: JSON.stringify(user.teams || []), + } + + const encryptedData: Record = {} + for (const [key, value] of Object.entries(data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, namespace, value) + } + + manifests.push({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: userId, + namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: userId, namespace }, + type: 'Opaque', + }, + }, + }) + } + + d.info(`Created ${manifests.length} individual user SealedSecret manifests`) + return manifests +} + export const bootstrapSealedSecrets = async ( secrets: Record, envDir: string, @@ -644,6 +791,7 @@ export const bootstrapSealedSecrets = async ( buildSecretToNamespaceMap, createSealedSecretManifest, writeSealedSecretManifests, + createUserSealedSecretManifests, encryptSecretItem, }, ): Promise => { @@ -679,7 +827,31 @@ export const bootstrapSealedSecrets = async ( manifests.push(manifest) } - // 7. Write SealedSecret manifests to disk + // 7. Create individual user SealedSecrets in apl-users namespace + const { users } = secrets + if (Array.isArray(users) && users.length > 0) { + // The users in allSecrets are in processed format (with groups). + // We also need original user data (isPlatformAdmin, isTeamAdmin, teams) from allValues. + const originalUsers = get(allValues, 'users', []) as any[] + // Merge original user fields with processed users for complete SealedSecret data + const usersForSecrets = users.map((processedUser: any) => { + const originalUser = originalUsers.find((u: any) => u.email === processedUser.email) + return { + ...processedUser, + name: originalUser?.name || processedUser.name, + isPlatformAdmin: originalUser?.isPlatformAdmin ?? false, + isTeamAdmin: originalUser?.isTeamAdmin ?? false, + teams: originalUser?.teams || [], + } + }) + const userManifests = await deps.createUserSealedSecretManifests(usersForSecrets, pem, { + encryptSecretItem: deps.encryptSecretItem, + terminal: deps.terminal, + }) + manifests.push(...userManifests) + } + + // 8. Write SealedSecret manifests to disk // Note: These manifests are applied later during install, after the sealed-secrets // controller is deployed and the SealedSecret CRD is available. await deps.writeSealedSecretManifests(manifests, envDir) diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index 671fc88d87..b8df5ae921 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -53,6 +53,9 @@ jest.mock('../common/utils', () => ({ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) +jest.mock('../common/sealed-secrets', () => ({ + aggregateUserSecrets: jest.fn().mockResolvedValue(undefined), +})) jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 8618955f3b..b631bb0fae 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -3,8 +3,9 @@ import { commit } from '../cmd/commit' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { GitRepoConfig } from '../common/git-config' -import { hfValues } from '../common/hf' import { waitTillGitRepoAvailable } from '../common/gitea' +import { hfValues } from '../common/hf' +import { aggregateUserSecrets } from '../common/sealed-secrets' import { ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -99,6 +100,10 @@ export class AplOperator { } else { await this.aplOps.apply() } + + // Aggregate individual user secrets into users-secrets for Keycloak + await aggregateUserSecrets() + this.d.info(`[${trigger}] Apply process completed`) await updateApplyState({ From 5dcded550dafa9dc043cb87f408ae5e03454006b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 13:59:23 +0100 Subject: [PATCH 24/71] feat: update user management --- src/common/sealed-secrets.test.ts | 9 +- src/common/sealed-secrets.ts | 115 +----------------- src/operator/apl-operator.test.ts | 3 - src/operator/apl-operator.ts | 4 - .../apl-keycloak-operator-raw.gotmpl | 5 - versions.yaml | 2 +- 6 files changed, 7 insertions(+), 131 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 01d610f144..9deedf76c1 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -200,7 +200,7 @@ describe('sealed-secrets', () => { expect(result[0].namespace).toBe('apl-secrets') }) - it('should serialize users array as single JSON value in users-secrets', async () => { + it('should skip users path (managed individually in apl-users namespace)', async () => { const secrets = { users: [ { @@ -219,11 +219,8 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - expect(result).toHaveLength(2) - const usersMapping = result.find((m) => m.secretName === 'users-secrets') - expect(usersMapping).toBeDefined() - expect(usersMapping!.namespace).toBe('apl-secrets') - expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) + expect(result).toHaveLength(1) + expect(result.find((m) => m.secretName === 'users-secrets')).toBeUndefined() }) it('should handle teamConfig dynamic paths in apl-secrets namespace', async () => { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index a454c2f7ca..2df49ff667 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -73,7 +73,6 @@ export const APP_NAMESPACE_MAP: Record = { dns: 'external-dns', obj: 'otomi', license: 'otomi', - users: 'keycloak', alerts: 'monitoring', cluster: 'cert-manager', } @@ -261,7 +260,6 @@ export const SECRET_NAME_MAP: Record = { dns: 'dns-secrets', obj: 'obj-storage-secrets', license: 'license-secrets', - users: 'users-secrets', alerts: 'alerts-secrets', cluster: 'cluster-secrets', } @@ -319,12 +317,11 @@ const deriveSecretName = (secretPath: string): string => { export const buildSecretToNamespaceMap = async ( secrets: Record, teams: string[], - allValues?: Record, + _allValues?: Record, deps = { getSchemaSecretsPaths }, ): Promise => { const secretPaths = await deps.getSchemaSecretsPaths(teams) const flat = flattenObject(secrets) - const allFlat = allValues ? flattenObject(allValues) : flat // Group by namespace + secretName const groupMap = new Map() @@ -332,21 +329,8 @@ export const buildSecretToNamespaceMap = async ( for (const secretPath of secretPaths) { // Skip SOPS-related paths if (secretPath.startsWith('kms.sops')) continue - // Handle 'users' path specially — serialize pre-processed users array as single JSON value - if (secretPath === 'users') { - const usersData = secrets.users - if (Array.isArray(usersData) && usersData.length > 0) { - const namespace = 'apl-secrets' - const secretName = 'users-secrets' - const groupKey = `${namespace}/${secretName}` - if (!groupMap.has(groupKey)) { - groupMap.set(groupKey, { namespace, secretName, data: {} }) - } - const mapping = groupMap.get(groupKey)! - mapping.data.usersJson = JSON.stringify(usersData) - } - continue - } + // Skip users path — user secrets are managed individually in apl-users namespace + if (secretPath === 'users') continue const namespace = resolveNamespace(secretPath) if (!namespace) continue @@ -627,99 +611,6 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi d.warn('Rollout status check timed out') } -/** - * Orchestrator: bootstrap sealed secrets for the platform. - * Replaces bootstrapSops(). - */ -/** - * Aggregate individual user K8s secrets from apl-users namespace into a single - * users-secrets Secret in apl-secrets namespace (usersJson key). - * This feeds the Keycloak Helm chart configuration. - * Called during the operator reconcile loop. - */ -export const aggregateUserSecrets = async (deps = { terminal, k8s, b64enc }): Promise => { - const d = deps.terminal(`common:${cmdName}:aggregateUserSecrets`) - const namespace = 'apl-users' - const targetNamespace = 'apl-secrets' - const targetSecretName = 'users-secrets' - - try { - const res: any = await deps.k8s.core().listNamespacedSecret({ namespace }) - const users: any[] = [] - - for (const item of res.items || []) { - if (item.type !== 'Opaque') continue - if (!item.data?.email) continue - - const decoded: Record = {} - for (const [key, value] of Object.entries(item.data as Record)) { - decoded[key] = Buffer.from(value, 'base64').toString('utf-8') - } - - // Build user in keycloak-operator format (with groups) - const groups: string[] = [] - if (decoded.isPlatformAdmin === 'true') groups.push('platform-admin') - if (decoded.isTeamAdmin === 'true') groups.push('team-admin') - const teams = decoded.teams ? JSON.parse(decoded.teams) : [] - for (const team of teams) groups.push(`team-${team}`) - - users.push({ - email: decoded.email, - firstName: decoded.firstName || '', - lastName: decoded.lastName || '', - initialPassword: decoded.initialPassword || '', - groups, - }) - } - - if (users.length === 0) { - d.info('No user secrets found in apl-users namespace, skipping aggregation') - return - } - - const usersJson = JSON.stringify(users) - - // Create or update the users-secrets Secret in apl-secrets - await ensureNamespaceExists(targetNamespace) - - try { - await deps.k8s.core().readNamespacedSecret({ name: targetSecretName, namespace: targetNamespace }) - // Secret exists, patch it - await deps.k8s.core().patchNamespacedSecret( - { - name: targetSecretName, - namespace: targetNamespace, - body: { - data: { usersJson: deps.b64enc(usersJson) }, - }, - }, - setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), - ) - } catch (error) { - if (error instanceof ApiException && error.code === 404) { - // Secret doesn't exist, create it - await deps.k8s.core().createNamespacedSecret({ - namespace: targetNamespace, - body: { - metadata: { - name: targetSecretName, - namespace: targetNamespace, - }, - type: 'Opaque', - data: { usersJson: deps.b64enc(usersJson) }, - }, - }) - } else { - throw error - } - } - - d.info(`Aggregated ${users.length} user(s) into ${targetNamespace}/${targetSecretName}`) - } catch (error) { - d.warn(`Failed to aggregate user secrets: ${error instanceof Error ? error.message : error}`) - } -} - /** * Create individual SealedSecret manifests for each user in the apl-users namespace. * Each user gets their own SealedSecret with all fields encrypted. diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index b8df5ae921..671fc88d87 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -53,9 +53,6 @@ jest.mock('../common/utils', () => ({ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) -jest.mock('../common/sealed-secrets', () => ({ - aggregateUserSecrets: jest.fn().mockResolvedValue(undefined), -})) jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index b631bb0fae..c9a0675169 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -5,7 +5,6 @@ import { env } from '../common/envalid' import { GitRepoConfig } from '../common/git-config' import { waitTillGitRepoAvailable } from '../common/gitea' import { hfValues } from '../common/hf' -import { aggregateUserSecrets } from '../common/sealed-secrets' import { ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -101,9 +100,6 @@ export class AplOperator { await this.aplOps.apply() } - // Aggregate individual user secrets into users-secrets for Keycloak - await aggregateUserSecrets() - this.d.info(`[${trigger}] Apply process completed`) await updateApplyState({ diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 2516d98edf..43763a08fa 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -41,7 +41,6 @@ resources: KEYCLOAK_ADMIN: {{ $k.adminUsername }} KEYCLOAK_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' KEYCLOAK_CLIENT_SECRET: '{{ "{{ .idpClientSecret | toString }}" }}' - USERS: '{{ "{{ .usersJson | toString }}" }}' {{- if $v.otomi.hasExternalIDP }} IDP_CLIENT_ID: '{{ "{{ .oidcClientID | toString }}" }}' IDP_CLIENT_SECRET: '{{ "{{ .oidcClientSecret | toString }}" }}' @@ -55,10 +54,6 @@ resources: remoteRef: key: keycloak-secrets property: idp_clientSecret - - secretKey: usersJson - remoteRef: - key: users-secrets - property: usersJson {{- if $v.otomi.hasExternalIDP }} - secretKey: oidcClientID remoteRef: diff --git a/versions.yaml b/versions.yaml index 29fc9f62a2..419c83400d 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: APL-523 console: main consoleLogin: main -tasks: APL-1476-1 +tasks: APL-523 tools: 2.10.7 From 85d4a32038d3416d772677ba49080a14677d1492 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:17:39 +0100 Subject: [PATCH 25/71] fix: create initial platform admin user --- src/cmd/bootstrap.test.ts | 5 ++++- src/cmd/bootstrap.ts | 4 +++- src/cmd/commit.ts | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 44e2684cda..2c68b1097e 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -268,7 +268,10 @@ describe('Bootstrapping values', () => { }) expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], + users: [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, + ], }) }) it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 06842b9486..608627e43b 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -212,7 +212,9 @@ export const processValues = async ( // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return { originalInput, allSecrets } + // Include users (with name/UUID) on originalInput for bootstrapSealedSecrets to find them. + // getUsers() may return a detached array when originalInput had no 'users' key initially. + return { originalInput: { ...originalInput, users }, allSecrets } } // create file structure based on file entry diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index a497aa8290..cebef6c6ec 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -169,23 +169,30 @@ export const commit = async ( } export async function initialSetupData(): Promise { + const d = terminal(`cmd:${cmdName}:initialSetupData`) const values = (await hfValues()) as Record const { domainSuffix } = values.cluster const { hasExternalIDP } = values.otomi const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' if (!hasExternalIDP) { - // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) - const usersSecret = await getK8sSecret('users-secrets', 'apl-secrets') + // Read the platform admin's initialPassword from individual user secrets in apl-users namespace let platformAdminPassword = '' - if (usersSecret?.usersJson) { - // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string - const users = Array.isArray(usersSecret.usersJson) - ? usersSecret.usersJson - : JSON.parse(String(usersSecret.usersJson)) + try { + const res: any = await k8s.core().listNamespacedSecret({ namespace: 'apl-users' }) const defaultEmail = `platform-admin@${domainSuffix}` - const platformAdmin = users.find((u: any) => u.email === defaultEmail) - platformAdminPassword = platformAdmin?.initialPassword ?? '' + for (const item of res.items || []) { + if (item.type !== 'Opaque' || !item.data?.email) continue + const email = Buffer.from(item.data.email, 'base64').toString('utf-8') + if (email === defaultEmail) { + platformAdminPassword = item.data.initialPassword + ? Buffer.from(item.data.initialPassword, 'base64').toString('utf-8') + : '' + break + } + } + } catch (error) { + d.warn(`Failed to read user secrets from apl-users: ${error instanceof Error ? error.message : error}`) } return { domainSuffix, From 4951838c27832034c56e37c5833da8c1831eafbc Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:48:43 +0100 Subject: [PATCH 26/71] fix: create initial platform admin user --- src/cmd/commit.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index cebef6c6ec..eaa1b85cb0 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,6 +1,7 @@ import retry from 'async-retry' import { bootstrapGit, setIdentity } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' +import { DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -176,23 +177,17 @@ export async function initialSetupData(): Promise { const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' if (!hasExternalIDP) { - // Read the platform admin's initialPassword from individual user secrets in apl-users namespace + // Read the platform admin's initialPassword from the generated passwords secret let platformAdminPassword = '' try { - const res: any = await k8s.core().listNamespacedSecret({ namespace: 'apl-users' }) + const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi') + const allSecrets = secretData?.[DEPLOYMENT_PASSWORDS_SECRET] + const users = allSecrets?.users || [] const defaultEmail = `platform-admin@${domainSuffix}` - for (const item of res.items || []) { - if (item.type !== 'Opaque' || !item.data?.email) continue - const email = Buffer.from(item.data.email, 'base64').toString('utf-8') - if (email === defaultEmail) { - platformAdminPassword = item.data.initialPassword - ? Buffer.from(item.data.initialPassword, 'base64').toString('utf-8') - : '' - break - } - } + const platformAdmin = users.find((u: any) => u.email === defaultEmail) + platformAdminPassword = platformAdmin?.initialPassword || '' } catch (error) { - d.warn(`Failed to read user secrets from apl-users: ${error instanceof Error ? error.message : error}`) + d.warn(`Failed to read platform admin credentials: ${error instanceof Error ? error.message : error}`) } return { domainSuffix, From 1fa5be8be6ddb1ce6dbb4b54f688602efc4d1dd0 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:38:38 +0100 Subject: [PATCH 27/71] revert: sops changes for the migration --- .github/workflows/integration.yml | 13 ++++++++++ .github/workflows/main.yml | 2 +- .github/workflows/otomi-tools-build-push.yaml | 7 +----- Dockerfile | 4 +-- helmfile.d/snippets/sops-env.gotmpl | 24 ++++++++++++++++++ src/common/bootstrap.ts | 5 ++++ src/common/envalid.ts | 1 + src/operator/installer.test.ts | 25 ++++++++++++++++++- src/operator/installer.ts | 19 ++++++++++++-- tools/Dockerfile | 22 ++++++++++++++++ versions.yaml | 2 +- 11 files changed, 111 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d3b5bf0f7a..f1cb7989a9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,6 +22,10 @@ on: description: 'Select Domain Zone' type: string default: DNS-Integration + kms: + description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' + type: string + default: age certificate: description: 'Select certificate issuer' type: string @@ -81,6 +85,13 @@ on: - Zone-2 - Random - DNS-Integration + kms: + type: choice + description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? + options: + - age + - no_kms + default: no_kms certificate: type: choice description: Select certificate issuer @@ -131,6 +142,7 @@ jobs: echo 'install_profile: ${{ inputs.install_profile }}' echo 'lke_tier: ${{ inputs.lke_tier }}' echo 'kubernetes_version: ${{ inputs.kubernetes_version }}' + echo 'kms: ${{ inputs.kms }}' echo 'domain_zone: ${{ inputs.domain_zone }}' echo 'certificate: ${{ inputs.certificate }}' echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' @@ -302,6 +314,7 @@ jobs: additional_args="" [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml + [[ '${{ inputs.kms }}' == 'age' ]] && additional_args+=" --set kms.sops.provider=age" [[ '${{ inputs.is_pre_installed }}' == 'true' ]] && additional_args+=" --set otomi.isPreInstalled=true" if [[ '${{ inputs.disableORCS }}' == 'true' ]]; then additional_args+=" --set otomi.useORCS=false" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02a3db4543..feb4919bc9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,7 @@ jobs: if: always() && contains(needs.release.result, 'success') && !github.event.act runs-on: ubuntu-22.04 container: - image: linode/apl-tools:v2.10.7 + image: linode/apl-tools:v2.10.6 options: --user 0 # See https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user steps: - name: Checkout diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3d38c4bc3f..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode @@ -71,11 +70,7 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - - # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored - NEW_VERSION="v2.10.7" - echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV - echo "Overriding NEW_VERSION=${NEW_VERSION}" + echo "No need to bump the version. Will skip next steps." - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/Dockerfile b/Dockerfile index 58a4f88252..caa68fc085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM linode/apl-tools:v2.10.7 AS ci +FROM linode/apl-tools:v2.10.6 AS ci ENV APP_HOME=/home/app/stack @@ -36,7 +36,7 @@ FROM ci AS clean # below command removes the packages specified in devDependencies and set NODE_ENV to production RUN npm prune --production -FROM linode/apl-tools:v2.10.7 AS prod +FROM linode/apl-tools:v2.10.6 AS prod ARG APPS_REVISION='' ENV APP_HOME=/home/app/stack ENV ENV_DIR=/home/app/stack/env diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index e69de29bb2..aa9e2ff866 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -0,0 +1,24 @@ +{{- with . | get "azure" nil }} +AZURE_CLIENT_ID: {{ .clientId }} +AZURE_CLIENT_SECRET: {{ .clientSecret }} +{{- with . | get "tenantId" nil }} +AZURE_TENANT_ID: {{ . }}{{ end }} +{{- with . | get "environment" nil }} +AZURE_ENVIRONMENT: {{ . }}{{ end }} +{{- end }} +{{- with . | get "aws" nil }} +AWS_ACCESS_KEY_ID: {{ .accessKey }} +AWS_SECRET_ACCESS_KEY: {{ .secretKey }} +{{- with . | get "region" nil }} +AWS_REGION: {{ . }}{{ end }} +{{- end }} +{{- with . | get "age" nil }} +SOPS_AGE_KEY: {{ .privateKey }} +{{- end }} +{{- with . | get "google" nil }} +GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' +{{- with . | get "project" nil }} +GOOGLE_PROJECT: {{ . }}{{ end }} +{{- with . | get "region" nil }} +GOOGLE_REGION: {{ . }}{{ end }} +{{- end }} diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 6e493353eb..a897d1d976 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -90,5 +90,10 @@ export const bootstrapGit = async (inValues?: Record): Promise { }) describe('setEnvAndCreateSecrets', () => { - test('should complete without errors', async () => { + test('should set SOPS_AGE_KEY when secret exists', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) + await installer.setEnvAndCreateSecrets() + + expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') + expect(process.env.SOPS_AGE_KEY).toBe('AGE-SECRET-KEY-1ABC') + }) + + test('should not throw when secret does not exist', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() + }) + + test('should not throw when getK8sSecret fails', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('Not found')) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() + }) + + test('should handle secret without SOPS_AGE_KEY field', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ OTHER_KEY: 'value' }) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index c4616cebd9..df2d950b3d 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,7 +1,7 @@ import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -122,6 +122,21 @@ export class Installer { } public async setEnvAndCreateSecrets(): Promise { - this.d.debug('Environment setup complete (SOPS removed, using SealedSecrets + ESO)') + this.d.debug('Setting up environment') + await this.setupSopsEnvironment() + } + + private async setupSopsEnvironment(): Promise { + try { + const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') + if (!aplSopsSecret?.SOPS_AGE_KEY) { + this.d.info('SOPS_AGE_KEY not found — cluster may already use SealedSecrets') + return + } + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.info('SOPS environment configured') + } catch (error) { + this.d.info('Could not read apl-sops-secrets — cluster may already use SealedSecrets') + } } } diff --git a/tools/Dockerfile b/tools/Dockerfile index 4f22ba1053..4fad532e80 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -9,6 +9,12 @@ ARG KUBECTL_VERSION=1.34.2 ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 +# https://github.com/jkroepke/helm-secrets/releases +ARG HELM_SECRETS_VERSION=4.7.4 +# https://github.com/getsops/sops/releases +ARG SOPS_VERSION=3.11.0 +# https://github.com/FiloSottile/age/releases +ARG AGE_VERSION=1.2.1 # https://github.com/noqcks/gucci/releases ARG GUCCI_VERSION=1.9.0 # https://github.com/yannh/kubeconform/releases/ @@ -30,6 +36,7 @@ RUN apt-get update && apt-get install -y \ curl \ apache2-utils \ ca-certificates \ + coreutils \ git \ locales \ rsync && \ @@ -43,6 +50,9 @@ RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq mv jq-linux-${TARGETARCH} /usr/bin/jq && \ chmod +x /usr/bin/jq +# yq +COPY --from=mikefarah/yq:4 /usr/bin/yq /usr/bin/yq + RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -65,10 +75,22 @@ RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo chmod +x kubectl-cnpg && \ rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz +# sops +ADD https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64 sops +RUN chmod +x sops + +# age +ADD https://github.com/FiloSottile/age/releases/download/v${AGE_VERSION}/age-v${AGE_VERSION}-linux-amd64.tar.gz age.tar.gz +RUN tar -zxvf age.tar.gz age/age age/age-keygen --strip-components=1 && \ + chmod +x age age-keygen && \ + rm -rf age.tar.gz + # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_DIFF_VERSION} +RUN echo "exec \$*" > /usr/bin/sudo && chmod +x /usr/bin/sudo +RUN helm plugin install https://github.com/jkroepke/helm-secrets --version ${HELM_SECRETS_VERSION} # helmfile ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp diff --git a/versions.yaml b/versions.yaml index 419c83400d..d722673428 100644 --- a/versions.yaml +++ b/versions.yaml @@ -2,4 +2,4 @@ api: APL-523 console: main consoleLogin: main tasks: APL-523 -tools: 2.10.7 +tools: main From bbdeaff4542fb403d0afce8f1e3758aed7b855e2 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:20:13 +0100 Subject: [PATCH 28/71] test: platform secrets migration --- src/cmd/migrate.test.ts | 205 +++++++++++++++++++++++++++++++++++++++- src/cmd/migrate.ts | 119 +++++++++++++++++++++++ values-changes.yaml | 5 + 3 files changed, 328 insertions(+), 1 deletion(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 3d33a34e13..07e2123055 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,5 +1,14 @@ +import { existsSync, rmSync } from 'fs' import { globSync } from 'glob' -import { applyChanges, Changes, filterChanges, getBuildName, policiesMigration } from 'src/cmd/migrate' +import { + applyChanges, + Changes, + filterChanges, + getBuildName, + policiesMigration, + removeSopsArtifacts, + sopsMigration, +} from 'src/cmd/migrate' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { getFileMap } from '../common/repo' @@ -13,6 +22,7 @@ jest.mock('../common/k8s') jest.mock('../common/values') jest.mock('../common/yargs') jest.mock('../common/utils') +jest.mock('../common/sealed-secrets') jest.mock('zx') jest.mock('@linode/kubeseal-encrypt') jest.mock('fs', () => ({ @@ -904,3 +914,196 @@ describe('setDefaultAplCatalog migration', () => { ) }, 20000) }) + +describe('sopsMigration', () => { + const mockTerminal = terminal + const mockExistsSync = jest.fn() + const mockGlobSync = jest.fn() + const mockGetExistingSealedSecretsCert = jest.fn() + const mockGetPemFromCertificate = jest.fn() + const mockGenerateSealedSecretsKeyPair = jest.fn() + const mockCreateSealedSecretsKeySecret = jest.fn() + const mockBuildSecretToNamespaceMap = jest.fn() + const mockCreateSealedSecretManifest = jest.fn() + const mockCreateUserSealedSecretManifests = jest.fn() + const mockWriteSealedSecretManifests = jest.fn() + const mockGetSchemaSecretsPaths = jest.fn() + const mockRemoveSopsArtifacts = jest.fn() + + const makeDeps = () => ({ + existsSync: mockExistsSync, + globSync: mockGlobSync, + terminal: mockTerminal, + getExistingSealedSecretsCert: mockGetExistingSealedSecretsCert, + getPemFromCertificate: mockGetPemFromCertificate, + generateSealedSecretsKeyPair: mockGenerateSealedSecretsKeyPair, + createSealedSecretsKeySecret: mockCreateSealedSecretsKeySecret, + buildSecretToNamespaceMap: mockBuildSecretToNamespaceMap, + createSealedSecretManifest: mockCreateSealedSecretManifest, + createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, + writeSealedSecretManifests: mockWriteSealedSecretManifests, + getSchemaSecretsPaths: mockGetSchemaSecretsPaths, + removeSopsArtifacts: mockRemoveSopsArtifacts, + }) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should skip when no .sops.yaml exists', async () => { + mockExistsSync.mockReturnValue(false) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).not.toHaveBeenCalled() + }) + + it('should only clean up when manifests already exist', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue(['/some/manifest.yaml']) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should run full migration path', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue(undefined) + mockGenerateSealedSecretsKeyPair.mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem' }) + mockCreateSealedSecretsKeySecret.mockResolvedValue(undefined) + mockGetPemFromCertificate.mockReturnValue('spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([ + { namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { adminPassword: 'pass' } }, + ]) + mockCreateSealedSecretManifest.mockResolvedValue({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'gitea-secrets', namespace: 'apl-secrets', annotations: {} }, + spec: { encryptedData: { adminPassword: 'encrypted' }, template: {} }, + }) + mockCreateUserSealedSecretManifests.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword']) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + apps: { gitea: { adminPassword: 'pass' } }, + } + + await sopsMigration(values, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(mockBuildSecretToNamespaceMap).toHaveBeenCalled() + expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) + expect(mockWriteSealedSecretManifests).toHaveBeenCalled() + expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + // Secrets should be stripped from values (in-place mutation) + expect(values.apps.gitea.adminPassword).toBeUndefined() + }) + + it('should use existing certificate when available', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('existing-cert-pem') + mockGetPemFromCertificate.mockReturnValue('existing-spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).not.toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).not.toHaveBeenCalled() + expect(mockGetPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should handle users', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockCreateUserSealedSecretManifests.mockResolvedValue([ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'user1', namespace: 'apl-users' }, + }, + ]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + users: [{ name: 'user1', email: 'user1@example.com' }], + } + + await sopsMigration(values, makeDeps()) + + expect(mockCreateUserSealedSecretManifests).toHaveBeenCalledWith( + [{ name: 'user1', email: 'user1@example.com' }], + 'pem', + ) + }) + + it('should handle empty secrets gracefully', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockCreateSealedSecretManifest).not.toHaveBeenCalled() + expect(mockWriteSealedSecretManifests).toHaveBeenCalledWith([], env.ENV_DIR) + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + }) +}) + +describe('removeSopsArtifacts', () => { + it('should remove .sops.yaml and all secrets files', () => { + const mockExistsSync = jest.fn().mockReturnValue(true) + const mockRmSync = jest.fn() + const mockGlobSync = jest + .fn() + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`) + }) + + it('should skip .sops.yaml removal when it does not exist', () => { + const mockExistsSync = jest.fn().mockReturnValue(false) + const mockRmSync = jest.fn() + const mockGlobSync = jest.fn().mockReturnValue([]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).not.toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + }) +}) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index f5c9cf5121..58783d3c17 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,6 +20,17 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' +import { + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + createUserSealedSecretManifests, + generateSealedSecretsKeyPair, + getExistingSealedSecretsCert, + getPemFromCertificate, + SealedSecretManifest, + writeSealedSecretManifests, +} from '../common/sealed-secrets' import { ARGOCD_APP_PARAMS } from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' @@ -740,6 +751,113 @@ const setDefaultAplCatalog = async (values: Record): Promise set(values, 'catalogs.default', defaultCatalog) } +export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, terminal }): void => { + const d = deps.terminal(`cmd:${cmdName}:removeSopsArtifacts`) + + // Remove .sops.yaml — makes encrypt()/decrypt() no-ops + const sopsConfigPath = `${env.ENV_DIR}/.sops.yaml` + if (deps.existsSync(sopsConfigPath)) { + deps.rmSync(sopsConfigPath) + d.info('Removed .sops.yaml') + } + + // Remove SOPS-encrypted files + const sopsEncrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml`, { dot: false }) + for (const f of sopsEncrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } + + // Remove SOPS-decrypted files + const sopsDecrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml.dec`, { dot: false }) + for (const f of sopsDecrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } +} + +export const sopsMigration = async ( + values: Record, + deps = { + existsSync, + globSync, + terminal, + getExistingSealedSecretsCert, + getPemFromCertificate, + generateSealedSecretsKeyPair, + createSealedSecretsKeySecret, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createUserSealedSecretManifests, + writeSealedSecretManifests, + getSchemaSecretsPaths, + removeSopsArtifacts, + }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:sopsMigration`) + + // Idempotency guard: no SOPS config means nothing to migrate + if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { + d.info('No .sops.yaml found, skipping SOPS migration') + return + } + + // Secondary guard: if manifests already exist, just clean up SOPS artifacts + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + if (existingManifests.length > 0) { + d.info('SealedSecret manifests already exist, only cleaning up SOPS artifacts') + deps.removeSopsArtifacts() + return + } + + d.info('Starting SOPS to SealedSecrets migration') + + // Get or generate sealed-secrets key + let pem: string + const existingCert = await deps.getExistingSealedSecretsCert() + if (existingCert) { + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } + + // Build secret-to-namespace mappings + const teams = Object.keys((values.teamConfig as Record) || {}) + const mappings = await deps.buildSecretToNamespaceMap(values, teams, values) + + // Create core SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping) + manifests.push(manifest) + } + + // Create user SealedSecret manifests + const users = values.users as any[] | undefined + if (Array.isArray(users) && users.length > 0) { + const userManifests = await deps.createUserSealedSecretManifests(users, pem) + manifests.push(...userManifests) + } + + // Write manifests to disk + await deps.writeSealedSecretManifests(manifests, env.ENV_DIR) + d.info(`Wrote ${manifests.length} SealedSecret manifests`) + + // Strip secrets from values (in-place mutation — writeValues() persists after return) + const secretPaths = await deps.getSchemaSecretsPaths(teams) + for (const path of secretPaths) { + unset(values, path) + } + + // Remove SOPS artifacts + deps.removeSopsArtifacts() + d.info('SOPS to SealedSecrets migration complete') +} + const customMigrationFunctions: Record = { networkPoliciesMigration, teamSettingsMigration, @@ -751,6 +869,7 @@ const customMigrationFunctions: Record = { workloadValuesMigration, setLokiStorageSchemaMigration, setDefaultAplCatalog, + sopsMigration, } /** diff --git a/values-changes.yaml b/values-changes.yaml index 52ee7013e5..54b771a686 100644 --- a/values-changes.yaml +++ b/values-changes.yaml @@ -446,3 +446,8 @@ changes: relocations: - apps.gitea.adminPassword: otomi.git.password - apps.gitea.adminUsername: otomi.git.username + - version: 56 + deletions: + - 'kms.sops' + customFunctions: + - sopsMigration From 44781ee97e092f78df9bec1676d735292ca00e9f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:05:55 +0100 Subject: [PATCH 29/71] fix: platform secrets migration --- src/cmd/migrate.test.ts | 4 +++- src/cmd/migrate.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 07e2123055..b76d50c3a5 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1072,13 +1072,14 @@ describe('sopsMigration', () => { }) describe('removeSopsArtifacts', () => { - it('should remove .sops.yaml and all secrets files', () => { + it('should remove .sops.yaml, secrets files, and user files', () => { const mockExistsSync = jest.fn().mockReturnValue(true) const mockRmSync = jest.fn() const mockGlobSync = jest .fn() .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`]) .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/users/some-uuid.yaml`]) removeSopsArtifacts({ existsSync: mockExistsSync, @@ -1090,6 +1091,7 @@ describe('removeSopsArtifacts', () => { expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`) expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/users/some-uuid.yaml`) }) it('should skip .sops.yaml removal when it does not exist', () => { diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 58783d3c17..d83297039c 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -774,6 +774,15 @@ export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, termi deps.rmSync(f) d.info(`Removed ${f}`) } + + // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/ns/apl-users/. + // These files may contain SOPS-encrypted data that was written by the "Write default values" step + // before SOPS decryption ran, contaminating the public YAML files with ENC[...] strings. + const userFiles = deps.globSync(`${env.ENV_DIR}/env/users/*.yaml`, { dot: false }) + for (const f of userFiles) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } } export const sopsMigration = async ( From d9a168acde412d674e588a4dcdd3272c9f509717 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 21:00:18 +0100 Subject: [PATCH 30/71] fix: platform secrets migration --- src/cmd/migrate.test.ts | 3 +++ src/cmd/migrate.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index b76d50c3a5..8387f31457 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -927,6 +927,7 @@ describe('sopsMigration', () => { const mockCreateSealedSecretManifest = jest.fn() const mockCreateUserSealedSecretManifests = jest.fn() const mockWriteSealedSecretManifests = jest.fn() + const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -942,6 +943,7 @@ describe('sopsMigration', () => { createSealedSecretManifest: mockCreateSealedSecretManifest, createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, writeSealedSecretManifests: mockWriteSealedSecretManifests, + applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -1002,6 +1004,7 @@ describe('sopsMigration', () => { expect(mockBuildSecretToNamespaceMap).toHaveBeenCalled() expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) expect(mockWriteSealedSecretManifests).toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() expect(mockRemoveSopsArtifacts).toHaveBeenCalled() // Secrets should be stripped from values (in-place mutation) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index d83297039c..b5b9c93361 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -21,6 +21,7 @@ import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' import { + applySealedSecretManifestsFromDir, buildSecretToNamespaceMap, createSealedSecretManifest, createSealedSecretsKeySecret, @@ -799,6 +800,7 @@ export const sopsMigration = async ( createSealedSecretManifest, createUserSealedSecretManifests, writeSealedSecretManifests, + applySealedSecretManifestsFromDir, getSchemaSecretsPaths, removeSopsArtifacts, }, @@ -856,6 +858,11 @@ export const sopsMigration = async ( await deps.writeSealedSecretManifests(manifests, env.ENV_DIR) d.info(`Wrote ${manifests.length} SealedSecret manifests`) + // Apply SealedSecret manifests to the cluster so the sealed-secrets controller + // can decrypt them into K8s Secrets before the apply step needs them. + d.info('Applying SealedSecret manifests to cluster') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + // Strip secrets from values (in-place mutation — writeValues() persists after return) const secretPaths = await deps.getSchemaSecretsPaths(teams) for (const path of secretPaths) { From c28e7d8ecb8bb4fcd7239b2fc9e814cac3edf867 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:18:15 +0100 Subject: [PATCH 31/71] fix: platform secrets migration --- src/cmd/migrate.test.ts | 3 +++ src/cmd/migrate.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 8387f31457..560a18ed58 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -928,6 +928,7 @@ describe('sopsMigration', () => { const mockCreateUserSealedSecretManifests = jest.fn() const mockWriteSealedSecretManifests = jest.fn() const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) + const mockRestartSealedSecretsController = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -944,6 +945,7 @@ describe('sopsMigration', () => { createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, writeSealedSecretManifests: mockWriteSealedSecretManifests, applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, + restartSealedSecretsController: mockRestartSealedSecretsController, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -1005,6 +1007,7 @@ describe('sopsMigration', () => { expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) expect(mockWriteSealedSecretManifests).toHaveBeenCalled() expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() expect(mockRemoveSopsArtifacts).toHaveBeenCalled() // Secrets should be stripped from values (in-place mutation) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index b5b9c93361..ee0f34ddd3 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -29,6 +29,7 @@ import { generateSealedSecretsKeyPair, getExistingSealedSecretsCert, getPemFromCertificate, + restartSealedSecretsController, SealedSecretManifest, writeSealedSecretManifests, } from '../common/sealed-secrets' @@ -801,6 +802,7 @@ export const sopsMigration = async ( createUserSealedSecretManifests, writeSealedSecretManifests, applySealedSecretManifestsFromDir, + restartSealedSecretsController, getSchemaSecretsPaths, removeSopsArtifacts, }, @@ -863,6 +865,11 @@ export const sopsMigration = async ( d.info('Applying SealedSecret manifests to cluster') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + // Restart the sealed-secrets controller so it picks up the migration-generated key. + // Without this, the controller uses its auto-generated key and cannot decrypt. + d.info('Restarting sealed-secrets controller to use migration key') + await deps.restartSealedSecretsController() + // Strip secrets from values (in-place mutation — writeValues() persists after return) const secretPaths = await deps.getSchemaSecretsPaths(teams) for (const path of secretPaths) { From 7b4574c05cb3913d5927652f2c19a963c606751c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:37:35 +0100 Subject: [PATCH 32/71] fix: platform secrets migration --- src/cmd/migrate.test.ts | 31 ++++++++++++++++++++++++++++++- src/cmd/migrate.ts | 16 +++++++++++++++- src/common/constants.ts | 2 +- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 560a18ed58..001ee67523 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -929,6 +929,7 @@ describe('sopsMigration', () => { const mockWriteSealedSecretManifests = jest.fn() const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) const mockRestartSealedSecretsController = jest.fn().mockResolvedValue(undefined) + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -946,6 +947,7 @@ describe('sopsMigration', () => { writeSealedSecretManifests: mockWriteSealedSecretManifests, applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, restartSealedSecretsController: mockRestartSealedSecretsController, + getK8sSecret: mockGetK8sSecret, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -954,15 +956,42 @@ describe('sopsMigration', () => { jest.clearAllMocks() }) - it('should skip when no .sops.yaml exists', async () => { + it('should skip when no .sops.yaml exists and no manifests on disk', async () => { mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue([]) await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() expect(mockRemoveSopsArtifacts).not.toHaveBeenCalled() }) + it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + it('should only clean up when manifests already exist', async () => { mockExistsSync.mockReturnValue(true) mockGlobSync.mockReturnValue(['/some/manifest.yaml']) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index ee0f34ddd3..f6a8d5ac64 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -803,14 +803,28 @@ export const sopsMigration = async ( writeSealedSecretManifests, applySealedSecretManifestsFromDir, restartSealedSecretsController, + getK8sSecret, getSchemaSecretsPaths, removeSopsArtifacts, }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:sopsMigration`) - // Idempotency guard: no SOPS config means nothing to migrate + // Idempotency guard: no SOPS config means migration already ran. + // However, if SealedSecret manifests exist on disk but the K8s Secrets are not yet + // decrypted (e.g. the operator was killed after writing manifests but before applying + // them, or the controller used its auto-generated key), re-apply them and restart + // the controller so subsequent steps can resolve the git password. if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + if (existingManifests.length > 0) { + const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + if (!platformSecret) { + d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + await deps.restartSealedSecretsController() + } + } d.info('No .sops.yaml found, skipping SOPS migration') return } diff --git a/src/common/constants.ts b/src/common/constants.ts index 43feae7157..f6252f3985 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -12,7 +12,7 @@ export const ARGOCD_APP_DEFAULT_SYNC_POLICY = { allowEmpty: false, selfHeal: true, }, - syncOptions: ['ServerSideApply=true'], + syncOptions: ['ServerSideApply=true', 'CreateNamespace=true'], } export interface ObjectMetadata { From 23b900f5608bbc3242909c72ede32e0c46cbff4d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:55:17 +0100 Subject: [PATCH 33/71] test: versions --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index d722673428..0fa5b279e6 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: APL-523 -console: main -consoleLogin: main +console: APL-523 +consoleLogin: APL-523 tasks: APL-523 tools: main From 1199673933dbe493f961bd9018ad87e9293813e2 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:58:41 +0100 Subject: [PATCH 34/71] fix: sealed secrets opaque type --- src/common/sealed-secrets.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 2df49ff667..40503068ed 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -394,7 +394,7 @@ export const createSealedSecretManifest = async ( template: { immutable: false, metadata: { name: mapping.secretName, namespace: mapping.namespace }, - type: 'Opaque', + type: 'kubernetes.io/opaque', }, }, } @@ -659,7 +659,7 @@ export const createUserSealedSecretManifests = async ( template: { immutable: false, metadata: { name: userId, namespace }, - type: 'Opaque', + type: 'kubernetes.io/opaque', }, }, }) From 2c6bc1f206a84c0cdd86a9a125ce7c5e252d5637 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 11:48:08 +0100 Subject: [PATCH 35/71] fix: sealed secrets opaque type test --- src/common/sealed-secrets.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 9deedf76c1..fc5e1a5052 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -321,7 +321,7 @@ describe('sealed-secrets', () => { expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') - expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.type).toBe('kubernetes.io/opaque') expect(result.spec.template.metadata.name).toBe('harbor-secrets') expect(result.spec.template.metadata.namespace).toBe('apl-secrets') }) From a0ed5251e6ae2acba8aab273ff5229a92901778f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:22:00 +0100 Subject: [PATCH 36/71] fix: installer tests --- src/operator/installer.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 2f1a1c189a..12d59e7ccb 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -37,6 +37,10 @@ jest.mock('../common/git-config', () => ({ getStoredGitRepoConfig: jest.fn(), })) +jest.mock('src/common/bootstrap', () => ({ + recoverFromGit: jest.fn().mockResolvedValue(undefined), +})) + jest.mock('src/cmd/traces', () => ({ runTraceCollectionLoop: jest.fn().mockResolvedValue(undefined), })) From aa3e3a1dad324934bd2df3da0e8b39b687d8c4df Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:56:13 +0100 Subject: [PATCH 37/71] feat: sealed secrets disaster recovery --- .cspell.json | 1 + src/operator/installer.test.ts | 205 ++++++++++++++++++++++++++++++++- src/operator/installer.ts | 55 ++++++++- src/operator/main.ts | 3 +- values-schema.yaml | 25 ++++ 5 files changed, 284 insertions(+), 5 deletions(-) diff --git a/.cspell.json b/.cspell.json index db8f8e9472..60b53049a1 100644 --- a/.cspell.json +++ b/.cspell.json @@ -103,6 +103,7 @@ "backoff", "basepath", "binzx", + "bitnami", "blackbox", "bootstrapper", "calico", diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 12d59e7ccb..e12f58e6d6 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -1,5 +1,7 @@ +import { ApiException } from '@kubernetes/client-node' import * as gitConfig from '../common/git-config' import * as k8s from '../common/k8s' +import * as utils from '../common/utils' import { AplOperations } from './apl-operations' import { Installer } from './installer' @@ -28,11 +30,21 @@ jest.mock('../common/k8s', () => ({ createUpdateConfigMap: jest.fn(), createUpdateGenericSecret: jest.fn(), deletePendingHelmReleases: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), k8s: { core: jest.fn(), }, })) +jest.mock('../common/utils', () => ({ + ...jest.requireActual('../common/utils'), + loadYaml: jest.fn(), +})) + +jest.mock('../common/envalid', () => ({ + env: { VALUES_INPUT: '/tmp/test-values.yaml' }, +})) + jest.mock('../common/git-config', () => ({ getStoredGitRepoConfig: jest.fn(), })) @@ -58,7 +70,9 @@ describe('Installer', () => { jest.clearAllMocks() jest.useFakeTimers() - mockCoreApi = {} + mockCoreApi = { + createNamespacedSecret: jest.fn().mockResolvedValue(undefined), + } ;(k8s.k8s.core as jest.Mock).mockReturnValue(mockCoreApi) mockAplOps = { @@ -372,4 +386,193 @@ describe('Installer', () => { await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) }) + + describe('applyRecoveryManifests', () => { + test('should create secrets from manifest items', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + apiVersion: 'v1', + kind: 'List', + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key2', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydDI=', 'tls.key': 'a2V5Mg==' }, + }, + ], + }, + }, + }, + }) + + await installer.applyRecoveryManifests() + + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('sealed-secrets') + expect(k8s.ensureNamespaceExists).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith({ + namespace: 'sealed-secrets', + body: { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + }) + }) + + test('should handle 409 conflict (secret already exists)', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==' }, + }, + ], + }, + }, + }, + }) + + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(409, 'Conflict', {}, {})) + + await expect(installer.applyRecoveryManifests()).resolves.not.toThrow() + }) + + test('should skip when no manifests present', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { mode: 'recovery' }, + }) + + await installer.applyRecoveryManifests() + + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() + }) + + test('should skip when items array is empty', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { recovery: { manifests: { items: [] } } }, + }) + + await installer.applyRecoveryManifests() + + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() + }) + + test('should rethrow non-409 errors', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + data: {}, + }, + ], + }, + }, + }, + }) + + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(500, 'Internal Server Error', {}, {})) + + await expect(installer.applyRecoveryManifests()).rejects.toThrow() + }) + + test('should use default namespace when metadata.namespace is not set', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'my-secret' }, + data: { key: 'val' }, + }, + ], + }, + }, + }, + }) + + await installer.applyRecoveryManifests() + + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('default') + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith( + expect.objectContaining({ + namespace: 'default', + }), + ) + }) + }) + + describe('ensureRecoveryPrerequisites', () => { + test('should succeed without SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should succeed with SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should succeed with empty SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({}) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should throw when git config is missing', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('Git config not found')) + + await expect(installer.ensureRecoveryPrerequisites()).rejects.toThrow('Git config not found') + }) + }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 48b98cf4a3..03835a3faf 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,11 +1,21 @@ +import { ApiException } from '@kubernetes/client-node' import * as process from 'node:process' import { runTraceCollectionLoop } from 'src/cmd/traces' import { recoverFromGit } from 'src/common/bootstrap' import { APL_OPERATOR_NS, APL_OPERATOR_STATUS_CM } from 'src/common/constants' import { $ } from 'zx' import { terminal } from '../common/debug' +import { env } from '../common/envalid' import { getStoredGitRepoConfig, GIT_CONFIG_NAMESPACE } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { + createUpdateConfigMap, + deletePendingHelmReleases, + ensureNamespaceExists, + getK8sConfigMap, + getK8sSecret, + k8s, +} from '../common/k8s' +import { loadYaml } from '../common/utils' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -84,9 +94,48 @@ export class Installer { public async ensureRecoveryPrerequisites(): Promise { await getStoredGitRepoConfig() + // SOPS is optional — sealed-secrets clusters don't have it const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret || Object.keys(sopsSecret).length === 0) { - throw new Error('KMS/SOPS config not found in apl-sops-secrets secret') + if (sopsSecret && Object.keys(sopsSecret).length > 0) { + this.d.info('SOPS configuration found for recovery') + } else { + this.d.info('No SOPS configuration — sealed-secrets mode recovery') + } + } + + public async applyRecoveryManifests(): Promise { + const values = (await loadYaml(env.VALUES_INPUT)) as Record + const items = values?.installation?.recovery?.manifests?.items + if (!Array.isArray(items) || items.length === 0) { + this.d.info('No recovery manifests to apply') + return + } + + this.d.info(`Applying ${items.length} recovery manifest(s)`) + for (const item of items) { + const namespace = item.metadata?.namespace || 'default' + const name = item.metadata?.name + await ensureNamespaceExists(namespace) + + try { + await k8s.core().createNamespacedSecret({ + namespace, + body: { + apiVersion: item.apiVersion, + kind: item.kind, + metadata: { name, namespace, labels: item.metadata?.labels }, + type: item.type, + data: item.data, + }, + }) + this.d.info(`Created recovery secret ${namespace}/${name}`) + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + this.d.info(`Recovery secret ${namespace}/${name} already exists, skipping`) + } else { + throw error + } + } } } diff --git a/src/operator/main.ts b/src/operator/main.ts index 8dd20baf31..b76747770b 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -76,8 +76,9 @@ async function main(): Promise { if (isInstalled) { d.info('Installation already completed, skipping install steps') } else if (isRecoveryMode) { - d.info('Recovery mode enabled, checking external git and kms prerequisites') + d.info('Recovery mode enabled, checking prerequisites') await installer.ensureRecoveryPrerequisites() + await installer.applyRecoveryManifests() await installer.recoverFromGit() d.info('Recovery installation completed, switching installation mode to standard') await installer.resetRecoveryModeToStandard() diff --git a/values-schema.yaml b/values-schema.yaml index ce24bd18b5..45c5dfbafd 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -1565,6 +1565,31 @@ properties: - recovery - standard default: standard + recovery: + type: object + description: Recovery settings for disaster recovery scenarios. Only used when mode=recovery. + properties: + manifests: + type: object + description: > + K8s resource List containing recovery manifests (e.g., sealed-secrets TLS key pairs) + to apply during installation. Not stored in the values repository. + Export from old cluster: + kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key=active -o yaml + properties: + apiVersion: + type: string + kind: + type: string + items: + type: array + items: + type: object + additionalProperties: true + metadata: + type: object + additionalProperties: true + additionalProperties: true azure: description: Azure specific configuration. properties: From beffad6d4e48c24205a877ec15d8cc6da688f350 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:58:21 +0100 Subject: [PATCH 38/71] feat: improve users during bootstrap --- src/cmd/bootstrap.test.ts | 56 ++++++++++++------------------------ src/cmd/bootstrap.ts | 24 ++-------------- src/common/sealed-secrets.ts | 16 +---------- 3 files changed, 22 insertions(+), 74 deletions(-) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 2c68b1097e..21f8b043af 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -155,18 +155,8 @@ describe('Bootstrapping values', () => { { id: 'user1', initialPassword: 'existing-password' }, { id: 'user2', initialPassword: generatedPassword }, ] - // Pre-processed users (as stored in allSecrets for sealed secret generation) - const processedUsers = usersWithPasswords.map((u: any) => ({ - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - initialPassword: u.initialPassword, - groups: [ - ...(u.isPlatformAdmin ? ['platform-admin'] : []), - ...(u.isTeamAdmin ? ['team-admin'] : []), - ...(u.teams || []).map((t: string) => `team-${t}`), - ], - })) + // Users stored directly in allSecrets (keycloak-operator derives groups from raw fields) + const ca = { a: 'cert' } const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) @@ -207,7 +197,7 @@ describe('Bootstrapping values', () => { deps.getStoredClusterSecrets.mockReturnValue(secrets) deps.generateSecrets.mockReturnValue(allSecrets) await processValues(deps) - const expected = { ...allSecrets, users: processedUsers } + const expected = { ...allSecrets, users: usersWithPasswords } expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) @@ -230,7 +220,7 @@ describe('Bootstrapping values', () => { await processValues(deps) expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { ...mergedSecretsWithGenAndCa, - users: processedUsers, + users: usersWithPasswords, }) }) it('should not overwrite stored secrets', async () => { @@ -242,7 +232,7 @@ describe('Bootstrapping values', () => { expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { ...generatedSecrets, - users: processedUsers, + users: usersWithPasswords, }) }) it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { @@ -256,14 +246,13 @@ describe('Bootstrapping values', () => { deps.createCustomCA.mockReturnValue(ca) const res = await processValues(deps) // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) - // processedUsers adds groups:[] to each user via element-wise lodash merge expect(deps.writeValues).toHaveBeenNthCalledWith(1, { a: 'cert', gen: 'x', cluster: { name: 'bla', provider: 'dida' }, users: [ - { id: 'user1', initialPassword: 'existing-password', groups: [] }, - { id: 'user2', initialPassword: 'generated-password', groups: [] }, + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, ], }) expect(res.originalInput).toEqual({ @@ -276,9 +265,9 @@ describe('Bootstrapping values', () => { }) it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { // mergedForDisk = merge(originalInput, allSecrets, { users }) - // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + // allSecrets = merge(ca, storedSecrets, generatedSecrets) + users: usersWithPasswords const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { - users: processedUsers, + users: usersWithPasswords, }) const expectedDiskValues = merge( cloneDeep(secrets), @@ -308,39 +297,30 @@ describe('Bootstrapping values', () => { const result = await processValues(deps) // allSecrets should contain full unstripped secrets including pre-processed users expect(result.allSecrets).toEqual( - merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: usersWithPasswords }), ) }) - it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { - // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) - const storedProcessedUsers = [ + it('should store users as-is in allSecrets (keycloak-operator derives groups)', async () => { + const storedUsers = [ { email: 'platform-admin@example.com', firstName: 'platform', lastName: 'admin', initialPassword: 'existing-pass', - groups: ['platform-admin'], + isPlatformAdmin: true, + teams: ['dev'], }, ] deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedUsers }) deps.generateSecrets.mockReturnValue({}) deps.createCustomCA.mockReturnValue({}) - // getUsers returns the stored processed users (no isPlatformAdmin flag) - deps.getUsers.mockReturnValue(storedProcessedUsers) + deps.getUsers.mockReturnValue(storedUsers) const result = await processValues(deps) - // Groups should be preserved from existing data, not reset to [] - expect(result.allSecrets.users).toEqual([ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ]) + // Users stored directly — no groups transformation + expect(result.allSecrets.users).toEqual(storedUsers) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 608627e43b..64e8447129 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -182,27 +182,9 @@ export const processValues = async ( const allSecrets = merge(cloneDeep(caSecrets), cloneDeep(storedSecrets), cloneDeep(generatedSecrets)) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) - // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage - const processedUsers = users.map((user: any) => { - const groups: string[] = [] - if (user.isPlatformAdmin) groups.push('platform-admin') - if (user.isTeamAdmin) groups.push('team-admin') - for (const team of user.teams || []) groups.push(`team-${team}`) - // Preserve existing groups when boolean flags are absent (e.g., user recovered - // from stored secrets which uses the processed format without isPlatformAdmin/isTeamAdmin) - if (groups.length === 0 && Array.isArray(user.groups) && user.groups.length > 0) { - groups.push(...(user.groups as string[])) - } - return { - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - initialPassword: user.initialPassword, - groups, - } - }) - // Store processed users in allSecrets so they flow into sealed secret generation - allSecrets.users = processedUsers + // Store users in allSecrets for sealed secret generation + // The keycloak-operator derives groups from isPlatformAdmin/isTeamAdmin/teams directly + allSecrets.users = users // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) const mergedForDisk = merge(cloneDeep(originalInput), cloneDeep(allSecrets), cloneDeep({ users })) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 40503068ed..c63af2fc6b 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -721,21 +721,7 @@ export const bootstrapSealedSecrets = async ( // 7. Create individual user SealedSecrets in apl-users namespace const { users } = secrets if (Array.isArray(users) && users.length > 0) { - // The users in allSecrets are in processed format (with groups). - // We also need original user data (isPlatformAdmin, isTeamAdmin, teams) from allValues. - const originalUsers = get(allValues, 'users', []) as any[] - // Merge original user fields with processed users for complete SealedSecret data - const usersForSecrets = users.map((processedUser: any) => { - const originalUser = originalUsers.find((u: any) => u.email === processedUser.email) - return { - ...processedUser, - name: originalUser?.name || processedUser.name, - isPlatformAdmin: originalUser?.isPlatformAdmin ?? false, - isTeamAdmin: originalUser?.isTeamAdmin ?? false, - teams: originalUser?.teams || [], - } - }) - const userManifests = await deps.createUserSealedSecretManifests(usersForSecrets, pem, { + const userManifests = await deps.createUserSealedSecretManifests(users, pem, { encryptSecretItem: deps.encryptSecretItem, terminal: deps.terminal, }) From c16577efe9911a290fbc8d974e58cffa36195c25 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:25:07 +0100 Subject: [PATCH 39/71] fix: update sealed secret manifests path --- src/cmd/migrate.test.ts | 5 ++--- src/cmd/migrate.ts | 12 ++++++++---- src/common/sealed-secrets.test.ts | 6 ++++-- src/common/sealed-secrets.ts | 9 ++++----- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 001ee67523..26cfcf0b26 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,4 +1,3 @@ -import { existsSync, rmSync } from 'fs' import { globSync } from 'glob' import { applyChanges, @@ -970,7 +969,7 @@ describe('sopsMigration', () => { it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) @@ -982,7 +981,7 @@ describe('sopsMigration', () => { it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 322d44381b..9b344e78ae 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -709,7 +709,7 @@ const createCatalogSealedSecret = async ( }, }, } - const sealedSecretPath = `${env.ENV_DIR}/env/manifests/namespaces/argocd/${SEALED_SECRET_NAME}.yaml` + const sealedSecretPath = `${env.ENV_DIR}/env/manifests/namespaces/argocd/sealedsecrets/${SEALED_SECRET_NAME}.yaml` mkdirSync(dirname(sealedSecretPath), { recursive: true }) d.info(`Writing sealed secret to ${sealedSecretPath}`) writeFileSync(sealedSecretPath, objectToYaml(sealedSecret)) @@ -777,7 +777,7 @@ export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, termi d.info(`Removed ${f}`) } - // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/ns/apl-users/. + // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/namespaces/apl-users/sealedsecrets. // These files may contain SOPS-encrypted data that was written by the "Write default values" step // before SOPS decryption ran, contaminating the public YAML files with ENC[...] strings. const userFiles = deps.globSync(`${env.ENV_DIR}/env/users/*.yaml`, { dot: false }) @@ -816,7 +816,9 @@ export const sopsMigration = async ( // them, or the controller used its auto-generated key), re-apply them and restart // the controller so subsequent steps can resolve the git password. if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + dot: false, + }) if (existingManifests.length > 0) { const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') if (!platformSecret) { @@ -830,7 +832,9 @@ export const sopsMigration = async ( } // Secondary guard: if manifests already exist, just clean up SOPS artifacts - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + dot: false, + }) if (existingManifests.length > 0) { d.info('SealedSecret manifests already exist, only cleaning up SOPS artifacts') deps.removeSopsArtifacts() diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index fc5e1a5052..69dbc56f6c 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -375,9 +375,11 @@ describe('sealed-secrets', () => { await writeSealedSecretManifests(manifests, '/test', deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/apl-secrets', { recursive: true }) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/namespaces/apl-secrets/sealedsecrets', { + recursive: true, + }) expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/manifests/ns/apl-secrets/harbor-secrets.yaml', + '/test/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml', 'yaml-content', ) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index c63af2fc6b..84b6e8d3d1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -401,7 +401,7 @@ export const createSealedSecretManifest = async ( } /** - * Write SealedSecret manifests to the env/manifests/ns directory. + * Write SealedSecret manifests to the env/manifests/namespaces directory. */ export const writeSealedSecretManifests = async ( manifests: SealedSecretManifest[], @@ -411,8 +411,7 @@ export const writeSealedSecretManifests = async ( const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) for (const manifest of manifests) { - // /env/manifests/ns/argocd/ - const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` + const dir = `${envDir}/env/manifests/namespaces/${manifest.metadata.namespace}/sealedsecrets` await deps.mkdir(dir, { recursive: true }) const filePath = `${dir}/${manifest.metadata.name}.yaml` d.info(`Writing sealed secret to ${filePath}`) @@ -478,7 +477,7 @@ export const applySealedSecretManifests = async ( } /** - * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * Read and apply all SealedSecret manifests from the env/manifests/namespaces directory. * This should be called during install, after the sealed-secrets controller is deployed. */ export const applySealedSecretManifestsFromDir = async ( @@ -486,7 +485,7 @@ export const applySealedSecretManifestsFromDir = async ( deps = { terminal, readdir, readFile, existsSync }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) - const manifestsDir = join(envDir, 'env/manifests/ns') + const manifestsDir = join(envDir, 'env/manifests/namespaces') if (!deps.existsSync(manifestsDir)) { d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) From 0f4d223b94088739c5bafa69a624b3d257408b51 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:16:48 +0100 Subject: [PATCH 40/71] feat: update tests/fixtures for local dev env users --- src/common/sealed-secrets.ts | 10 +++++---- .../23d63558-49ed-48ba-bc28-8037a7236ddf.yaml | 22 +++++++++++++++++++ .../9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml | 22 +++++++++++++++++++ .../a83e20b7-474a-4262-a3ad-b09813364ece.yaml | 22 +++++++++++++++++++ .../bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 22 +++++++++++++++++++ ....23d63558-49ed-48ba-bc28-8037a7236ddf.yaml | 12 ---------- ....9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml | 12 ---------- ....a83e20b7-474a-4262-a3ad-b09813364ece.yaml | 10 --------- ....bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 10 --------- 9 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml delete mode 100644 tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml delete mode 100644 tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml delete mode 100644 tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml delete mode 100644 tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 84b6e8d3d1..ebfe22d2c0 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -501,15 +501,17 @@ export const applySealedSecretManifestsFromDir = async ( for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue const namespace = nsEntry.name - const nsDir = join(manifestsDir, namespace) + const sealedSecretsDir = join(manifestsDir, namespace, 'sealedsecrets') + + if (!deps.existsSync(sealedSecretsDir)) continue await ensureNamespaceExists(namespace) - // Read all YAML files in the namespace directory - const files = await deps.readdir(nsDir) + // Read all YAML files in the sealedsecrets subdirectory + const files = await deps.readdir(sealedSecretsDir) for (const file of files) { if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue - const filePath = join(nsDir, file) + const filePath = join(sealedSecretsDir, file) d.info(`Applying SealedSecret from ${filePath}`) try { diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml new file mode 100644 index 0000000000..cd965eb983 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users +spec: + encryptedData: + email: team@admin.com + firstName: team + lastName: admin + initialPassword: team-admin-password + isPlatformAdmin: 'false' + isTeamAdmin: 'true' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml new file mode 100644 index 0000000000..a9cabecfdb --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users +spec: + encryptedData: + email: team@member.com + firstName: team + lastName: member + initialPassword: team-member-password + isPlatformAdmin: 'false' + isTeamAdmin: 'false' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml new file mode 100644 index 0000000000..1ae170984a --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users +spec: + encryptedData: + email: platform@admin.com + firstName: platform + lastName: admin + initialPassword: platform-admin-password + isPlatformAdmin: 'true' + isTeamAdmin: 'true' + teams: '[]' + template: + immutable: false + metadata: + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml new file mode 100644 index 0000000000..358ce1b2fe --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: bc2fe5b1-835c-4998-ad64-e15d90062b16 + namespace: apl-users +spec: + encryptedData: + email: platform-admin@dev.linode-apl.net + firstName: platform + lastName: admin + initialPassword: 02LDWB#qzknkeF8f*m%% + isPlatformAdmin: 'true' + isTeamAdmin: 'false' + teams: '[]' + template: + immutable: false + metadata: + name: bc2fe5b1-835c-4998-ad64-e15d90062b16 + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml deleted file mode 100644 index 6576468c59..0000000000 --- a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 23d63558-49ed-48ba-bc28-8037a7236ddf -spec: - email: team@admin.com - firstName: team - initialPassword: team-admin-password - isPlatformAdmin: false - isTeamAdmin: true - lastName: admin - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml deleted file mode 100644 index 839049b33f..0000000000 --- a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 -spec: - email: team@member.com - firstName: team - initialPassword: team-member-password - isPlatformAdmin: false - isTeamAdmin: false - lastName: member - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml deleted file mode 100644 index a489101d4f..0000000000 --- a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: a83e20b7-474a-4262-a3ad-b09813364ece -spec: - email: platform@admin.com - firstName: platform - initialPassword: platform-admin-password - isPlatformAdmin: true - isTeamAdmin: true - lastName: admin diff --git a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml deleted file mode 100644 index e0fc2601d6..0000000000 --- a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 -spec: - email: platform-admin@dev.linode-apl.net - firstName: platform - lastName: admin - isPlatformAdmin: true - isTeamAdmin: false - initialPassword: 02LDWB#qzknkeF8f*m%% From 6aa05e68d0395a3015304e53a131b8252efe4c10 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:37:23 +0100 Subject: [PATCH 41/71] fix: update tests/fixtures for local dev env users --- .../bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml deleted file mode 100644 index 358ce1b2fe..0000000000 --- a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - annotations: - sealedsecrets.bitnami.com/namespace-wide: 'true' - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 - namespace: apl-users -spec: - encryptedData: - email: platform-admin@dev.linode-apl.net - firstName: platform - lastName: admin - initialPassword: 02LDWB#qzknkeF8f*m%% - isPlatformAdmin: 'true' - isTeamAdmin: 'false' - teams: '[]' - template: - immutable: false - metadata: - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 - namespace: apl-users - type: kubernetes.io/opaque From 1f5e7e00193f5bbbe270de8b20be406bb69b289b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 3 Mar 2026 10:14:16 +0100 Subject: [PATCH 42/71] fix: ci error --- src/cmd/migrate.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 9b344e78ae..fd993ff289 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -820,11 +820,15 @@ export const sopsMigration = async ( dot: false, }) if (existingManifests.length > 0) { - const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') - if (!platformSecret) { - d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') - await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) - await deps.restartSealedSecretsController() + try { + const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + if (!platformSecret) { + d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + await deps.restartSealedSecretsController() + } + } catch { + d.info('Could not reach K8s API to check secrets, skipping re-apply') } } d.info('No .sops.yaml found, skipping SOPS migration') From e5d7fedf878bc5a9a26c478e209db6706fbc858f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Mar 2026 23:14:31 +0100 Subject: [PATCH 43/71] feat: enhance sealed secrets management and update dependencies --- helmfile.d/helmfile-60.teams.yaml.gotmpl | 57 ++++++++++ helmfile.d/snippets/defaults.yaml | 2 +- src/cmd/install.test.ts | 30 +++--- src/cmd/install.ts | 40 ++++--- src/common/sealed-secrets.test.ts | 101 +++++++++++++++--- src/common/sealed-secrets.ts | 96 ++++++++--------- tools/Dockerfile | 5 +- values/apl-operator/apl-operator-raw.gotmpl | 4 +- values/apl-operator/apl-operator.gotmpl | 3 +- values/argocd/argocd-raw.gotmpl | 18 ++++ values/cert-manager/cert-manager-raw.gotmpl | 15 +-- values/external-dns/external-dns.gotmpl | 2 +- .../external-secrets/external-secrets.gotmpl | 2 +- values/oauth2-proxy/oauth2-proxy-raw.gotmpl | 6 +- .../prometheus-operator-raw.gotmpl | 6 ++ .../prometheus-operator.gotmpl | 2 + 16 files changed, 268 insertions(+), 121 deletions(-) diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index ce769d6040..646f90166e 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -121,6 +121,8 @@ releases: url: http://loki-query-frontend-headless.monitoring:3101 basicAuth: true basicAuthUser: {{ $teamId }} + secureJsonData: + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} {{- if has "msteams" ($teamSettings | get "alerts.receivers" list) }} - name: prometheus-msteams-{{ $teamId }} installed: {{ $teamSettings | get "managedMonitoring.alertmanager" false }} @@ -179,6 +181,7 @@ releases: pipeline: otomi-task-teams values: - resources: + {{- if $teamSettings | get "managedMonitoring.grafana" false }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -202,6 +205,54 @@ releases: remoteRef: key: team-{{ $teamId }}-settings-secrets property: settings_password + {{- end }} + {{- if $teamSettings | get "managedMonitoring.grafana" false }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $v.apps.keycloak.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword + {{- end }} {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -238,5 +289,11 @@ releases: key: smtp-secrets property: auth_secret {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} {{- end }} {{- end }} diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index 1f60127045..0c1431b7be 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -1174,4 +1174,4 @@ environments: branch: main enabled: true versions: - specVersion: 55 + specVersion: 56 diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 5803bda66b..16704a0499 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -61,7 +61,7 @@ jest.mock('zx', () => { jest.mock('src/common/sealed-secrets', () => ({ applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), - buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + SECRET_NAME_MAP: {}, })) jest.mock('src/common/utils', () => ({ @@ -281,32 +281,28 @@ describe('Install command', () => { }) describe('error handling', () => { - test('should handle deployment state errors', async () => { - const error = new Error('Failed to get deployment state') - mockDeps.getDeploymentState.mockRejectedValueOnce(error) + test('should throw on deployment state errors', async () => { + mockDeps.getDeploymentState.mockRejectedValueOnce(new Error('Failed to get deployment state')) - expect(mockDeps.getDeploymentState).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get deployment state') }) - test('should handle image tag retrieval errors', async () => { - const error = new Error('Failed to get image tag') - mockDeps.getImageTagFromValues.mockRejectedValueOnce(error) + test('should throw on image tag retrieval errors', async () => { + mockDeps.getImageTagFromValues.mockRejectedValueOnce(new Error('Failed to get image tag')) - expect(mockDeps.getImageTagFromValues).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get image tag') }) - test('should handle helmfile errors', async () => { - const error = new Error('Helmfile execution failed') - mockDeps.hf.mockRejectedValueOnce(error) + test('should throw on helmfile errors during sealed-secrets deploy', async () => { + mockDeps.hf.mockRejectedValueOnce(new Error('Helmfile execution failed')) - expect(mockDeps.hf).toBeDefined() + await expect(installAll()).rejects.toThrow('Helmfile execution failed') }) - test('should handle CRDs deployment errors', async () => { - const error = new Error('CRDs deployment failed') - mockDeps.applyServerSide.mockRejectedValueOnce(error) + test('should throw on essential deployment failure', async () => { + mockDeps.deployEssential.mockResolvedValueOnce(false) - expect(mockDeps.applyServerSide).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to deploy essential manifests') }) }) }) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index edf04db93b..0e0a4c713a 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -19,8 +19,8 @@ import { } from 'src/common/k8s' import { applySealedSecretManifestsFromDir, - buildSecretToNamespaceMap, restartSealedSecretsController, + SECRET_NAME_MAP, } from 'src/common/sealed-secrets' import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' @@ -63,48 +63,46 @@ const retryInstallStep = async ( /** * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. - * Derives the list of secrets to wait for from schema x-secret fields via buildSecretToNamespaceMap(). + * Derives the expected secret names from schema x-secret fields + SECRET_NAME_MAP. */ const waitForSealedSecrets = async ( timeoutMs = 120000, intervalMs = 3000, - deps = { getK8sSecret, terminal, buildSecretToNamespaceMap, getSchemaSecretsPaths }, + deps = { getK8sSecret, terminal, getSchemaSecretsPaths }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) - // Build list of secrets to wait for from schema-driven mappings - // We pass empty secrets/teams since we just need the secret names and namespaces - const mappings = await deps.buildSecretToNamespaceMap({}, [], undefined, { - getSchemaSecretsPaths: deps.getSchemaSecretsPaths, - }) - - // Deduplicate by namespace/secretName - const secretsToWait = new Map() - for (const mapping of mappings) { - const key = `${mapping.namespace}/${mapping.secretName}` - if (!secretsToWait.has(key)) { - secretsToWait.set(key, { namespace: mapping.namespace, secretName: mapping.secretName }) + // Get all x-secret paths from schema and map them to K8s secret names + const secretPaths = await deps.getSchemaSecretsPaths([]) + const sortedPrefixes = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + const secretNames = new Set() + for (const path of secretPaths) { + for (const prefix of sortedPrefixes) { + if (path === prefix || path.startsWith(`${prefix}.`)) { + secretNames.add(SECRET_NAME_MAP[prefix]) + break + } } } - if (secretsToWait.size === 0) { + if (secretNames.size === 0) { d.info('No sealed secrets to wait for') return } - d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) + d.info(`Waiting for ${secretNames.size} sealed secrets to be decrypted: ${[...secretNames].join(', ')}`) await retry( async () => { const pending: string[] = [] - for (const { namespace, secretName } of secretsToWait.values()) { + for (const secretName of secretNames) { try { - const secret = await deps.getK8sSecret(secretName, namespace) + const secret = await deps.getK8sSecret(secretName, 'apl-secrets') if (!secret) { - pending.push(`${namespace}/${secretName}`) + pending.push(secretName) } } catch { - pending.push(`${namespace}/${secretName}`) + pending.push(secretName) } } diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 69dbc56f6c..25835a38a9 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -1,14 +1,17 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { - APP_NAMESPACE_MAP, + applySealedSecretManifests, bootstrapSealedSecrets, buildSecretToNamespaceMap, createSealedSecretManifest, createSealedSecretsKeySecret, + createUserSealedSecretManifests, generateSealedSecretsKeyPair, getPemFromCertificate, + restartSealedSecretsController, SECRET_NAME_MAP, + SealedSecretManifest, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -499,18 +502,6 @@ describe('sealed-secrets', () => { }) }) - describe('APP_NAMESPACE_MAP', () => { - it('should have expected mappings', () => { - expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') - expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') - expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') - expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') - expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') - expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') - expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') - }) - }) - describe('SECRET_NAME_MAP', () => { it('should have expected secret name mappings', () => { expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') @@ -555,4 +546,88 @@ describe('sealed-secrets', () => { expect(values.apps.gitea.adminPassword).toBe('secret') }) }) + + describe('applySealedSecretManifests', () => { + const makeMockManifest = (name: string, namespace: string): SealedSecretManifest => ({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name, + namespace, + }, + spec: { + encryptedData: { key: 'encrypted-value' }, + template: { + immutable: false, + metadata: { name, namespace }, + type: 'kubernetes.io/opaque', + }, + }, + }) + + it('should apply manifests successfully', async () => { + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await applySealedSecretManifests(manifests, { terminal }) + + const { k8s: mockK8s } = require('src/common/k8s') + expect(mockK8s.custom().createNamespacedCustomObject).toHaveBeenCalled() + }) + + it('should log error when manifests fail to apply', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + mockK8s.custom().createNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) + + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await applySealedSecretManifests(manifests, { terminal }) + // Should not throw, just log the error + }) + }) + + describe('restartSealedSecretsController', () => { + it('should succeed when deployment rolls out quickly', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + // Mock deployment that is already ready + mockK8s.app().readNamespacedDeployment.mockResolvedValue({ + spec: { replicas: 1 }, + status: { updatedReplicas: 1, availableReplicas: 1 }, + }) + + await restartSealedSecretsController({ terminal }) + + expect(mockK8s.app().patchNamespacedDeployment).toHaveBeenCalled() + }) + }) + + describe('createUserSealedSecretManifests', () => { + it('should create individual SealedSecret for each user', async () => { + const users = [ + { name: 'user1', email: 'user1@test.com', firstName: 'User', lastName: 'One', initialPassword: 'pass1' }, + { name: 'user2', email: 'user2@test.com', firstName: 'User', lastName: 'Two', initialPassword: 'pass2' }, + ] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(2) + expect(manifests[0].metadata.name).toBe('user1') + expect(manifests[0].metadata.namespace).toBe('apl-users') + expect(manifests[1].metadata.name).toBe('user2') + }) + + it('should skip users without id/name', async () => { + const users = [{ email: 'noname@test.com' }] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(0) + }) + }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index ebfe22d2c0..cd764bafda 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -52,30 +52,10 @@ export interface SealedSecretManifest { } /** - * Mapping from secret path prefix to target Kubernetes namespace. - * Dynamic entries like `teamConfig.{teamId}` are handled separately. + * All SealedSecrets are placed in the 'apl-secrets' namespace. + * ESO ClusterSecretStore reads from this namespace and distributes secrets to target namespaces. */ -export const APP_NAMESPACE_MAP: Record = { - 'apps.harbor': 'harbor', - 'apps.gitea': 'gitea', - 'apps.keycloak': 'keycloak', - 'apps.grafana': 'grafana', - 'apps.loki': 'monitoring', - 'apps.oauth2-proxy': 'istio-system', - 'apps.oauth2-proxy-redis': 'istio-system', - 'apps.prometheus': 'monitoring', - 'apps.otomi-api': 'otomi', - 'apps.cert-manager': 'cert-manager', - 'apps.kubeflow-pipelines': 'kfp', - otomi: 'otomi', - oidc: 'otomi', - smtp: 'otomi', - dns: 'external-dns', - obj: 'otomi', - license: 'otomi', - alerts: 'monitoring', - cluster: 'cert-manager', -} +const SEALED_SECRETS_NAMESPACE = 'apl-secrets' /** * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. @@ -220,21 +200,19 @@ export const createSealedSecretsKeySecret = async ( /** * Resolve the namespace for a given secret path. - * All core secrets go to 'apl-secrets' namespace for ESO access. - * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. + * All secrets go to 'apl-secrets' namespace for ESO ClusterSecretStore access. */ const resolveNamespace = (secretPath: string): string | undefined => { // Check for teamConfig dynamic paths - const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return 'apl-secrets' + if (secretPath.match(/^teamConfig\.[^.]+/)) { + return SEALED_SECRETS_NAMESPACE } - // Check if this path matches any known prefix - const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + // Check if this path matches any known secret name prefix + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) for (const prefix of sortedKeys) { if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return 'apl-secrets' + return SEALED_SECRETS_NAMESPACE } } @@ -355,6 +333,8 @@ export const buildSecretToNamespaceMap = async ( groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) ? flatKey.slice(groupPrefix.length + 1) : flatKey + // Skip empty relative paths (happens when flatKey === groupPrefix) + if (!relativePath) continue const dataKey = relativePath.replace(/\./g, '_') if (value !== undefined && value !== null && value !== '') { mapping.data[dataKey] = String(value) @@ -455,17 +435,21 @@ export const applySealedSecretManifests = async ( }) } catch (error) { if (error instanceof ApiException && error.code === 409) { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) + try { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + } catch (patchError) { + d.error(`Failed to patch SealedSecret ${manifest.metadata.name}: ${patchError}`) + } } else { d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) } @@ -529,18 +513,22 @@ export const applySealedSecretManifestsFromDir = async ( appliedCount += 1 } catch (error) { if (error instanceof ApiException && error.code === 409) { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - appliedCount += 1 + try { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + appliedCount += 1 + } catch (patchError) { + d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) + } } else { d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) } diff --git a/tools/Dockerfile b/tools/Dockerfile index 4fad532e80..af6f1b4836 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -11,7 +11,7 @@ ARG HELM_VERSION=3.19.2 ARG HELM_DIFF_VERSION=3.14.1 # https://github.com/jkroepke/helm-secrets/releases ARG HELM_SECRETS_VERSION=4.7.4 -# https://github.com/getsops/sops/releases +# https://github.com/mozilla/sops/releases ARG SOPS_VERSION=3.11.0 # https://github.com/FiloSottile/age/releases ARG AGE_VERSION=1.2.1 @@ -34,9 +34,10 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ + coreutils \ apache2-utils \ + apt-transport-https \ ca-certificates \ - coreutils \ git \ locales \ rsync && \ diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index bd18d18d0e..9b6652744f 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -4,14 +4,14 @@ resources: - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: gitea-credentials + name: apl-git-credentials spec: refreshInterval: 1h secretStoreRef: name: core-secrets-store kind: ClusterSecretStore target: - name: gitea-credentials + name: apl-git-credentials creationPolicy: Owner template: type: Opaque diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index bfe276488b..a54c848fa6 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,4 +24,5 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} - password: {{ $v.otomi.git | get "password" "" | quote }} + # Password intentionally empty — operator reads git credentials from K8s secret (apl-git-credentials) + password: "" diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 3d71ec6385..8c9966a6f9 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -96,6 +96,24 @@ resources: key: otomi-platform-secrets property: git_password {{- end }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: argocd-oidc-secret + namespace: argocd + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-secret + creationPolicy: Merge + data: + - secretKey: oidc.clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: v1 kind: Secret metadata: diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index e7f0df883f..e9fc23ce19 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -41,13 +41,7 @@ resources: secret: '{{ "{{ .secret | toString }}" }}' {{- end }} data: - - secretKey: secret - remoteRef: - key: dns-secrets - {{- if hasKey $p "google" }} - property: provider_google_serviceAccountKey - {{- else if hasKey $p "akamai" }} - property: provider_akamai_clientSecret + {{- if hasKey $p "akamai" }} - secretKey: access_token remoteRef: key: dns-secrets @@ -60,6 +54,12 @@ resources: remoteRef: key: dns-secrets property: provider_akamai_clientSecret + {{- else }} + - secretKey: secret + remoteRef: + key: dns-secrets + {{- if hasKey $p "google" }} + property: provider_google_serviceAccountKey {{- else if hasKey $p "azure-private-dns" }} property: provider_azure-private-dns_aadClientSecret {{- else if hasKey $p "azure" }} @@ -73,6 +73,7 @@ resources: {{- else if hasKey $p "linode" }} property: provider_linode_apiToken {{- end }} + {{- end }} {{- end }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret diff --git a/values/external-dns/external-dns.gotmpl b/values/external-dns/external-dns.gotmpl index e5f1d5e83c..88f1c64e75 100644 --- a/values/external-dns/external-dns.gotmpl +++ b/values/external-dns/external-dns.gotmpl @@ -214,7 +214,7 @@ env: extraVolumes: - name: google-service-account secret: - secretName: GOOGLE-DNS + secretName: google-dns extraVolumeMounts: - name: google-service-account mountPath: /etc/secrets/service-account diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl index 30ba6ada8c..0e16bcfaaf 100644 --- a/values/external-secrets/external-secrets.gotmpl +++ b/values/external-secrets/external-secrets.gotmpl @@ -1,5 +1,5 @@ {{- $v := .Values }} -{{- $app := $v.apps | get "sealed-secrets" }} +{{- $app := $v.apps | get "external-secrets" }} replicaCount: 1 diff --git a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl index 17696000bf..61450b8510 100644 --- a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl @@ -45,12 +45,16 @@ resources: data: client-id: {{ $k.idp.clientID }} client-secret: '{{ "{{ .clientSecret | toString }}" }}' - cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) }} + cookie-secret: '{{ "{{ .cookieSecret | toString }}" }}' data: - secretKey: clientSecret remoteRef: key: keycloak-secrets property: idp_clientSecret + - secretKey: cookieSecret + remoteRef: + key: oauth2-proxy-secrets + property: config_cookieSecret - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index b74bf5873b..b6ef5709b9 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -138,4 +138,10 @@ resources: key: smtp-secrets property: auth_secret {{- end }} + {{- if has "opsgenie" $receivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} {{- end }} diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index bc7a7199e5..6a92d8326d 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -247,6 +247,8 @@ grafana: {{- end }} basicAuth: true basicAuthUser: otomi-admin + secureJsonData: + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} {{- end }} {{- end }} admin: From be165b247307465f5d5ff86bdeac1db682e99bab Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Mar 2026 23:58:27 +0100 Subject: [PATCH 44/71] feat: update sealed secrets handling to return applied secrets list --- src/cmd/install.test.ts | 4 +--- src/cmd/install.ts | 44 ++++++++++++------------------------ src/common/sealed-secrets.ts | 14 +++++++----- 3 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 16704a0499..ba2c8ce625 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -59,15 +59,13 @@ jest.mock('zx', () => { }) jest.mock('src/common/sealed-secrets', () => ({ - applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue([]), restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), - SECRET_NAME_MAP: {}, })) jest.mock('src/common/utils', () => ({ ...jest.requireActual('src/common/utils'), rootDir: '/test/root', - getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), })) jest.mock('./commit', () => ({ diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 0e0a4c713a..9bc108f0cc 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -17,12 +17,8 @@ import { setDeploymentState, waitForCRD, } from 'src/common/k8s' -import { - applySealedSecretManifestsFromDir, - restartSealedSecretsController, - SECRET_NAME_MAP, -} from 'src/common/sealed-secrets' -import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' +import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' +import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' import { Argv, CommandModule } from 'yargs' @@ -63,46 +59,36 @@ const retryInstallStep = async ( /** * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. - * Derives the expected secret names from schema x-secret fields + SECRET_NAME_MAP. + * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ const waitForSealedSecrets = async ( + appliedSecrets: { namespace: string; secretName: string }[], timeoutMs = 120000, intervalMs = 3000, - deps = { getK8sSecret, terminal, getSchemaSecretsPaths }, + deps = { getK8sSecret, terminal }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) - // Get all x-secret paths from schema and map them to K8s secret names - const secretPaths = await deps.getSchemaSecretsPaths([]) - const sortedPrefixes = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - const secretNames = new Set() - for (const path of secretPaths) { - for (const prefix of sortedPrefixes) { - if (path === prefix || path.startsWith(`${prefix}.`)) { - secretNames.add(SECRET_NAME_MAP[prefix]) - break - } - } - } - - if (secretNames.size === 0) { + if (appliedSecrets.length === 0) { d.info('No sealed secrets to wait for') return } - d.info(`Waiting for ${secretNames.size} sealed secrets to be decrypted: ${[...secretNames].join(', ')}`) + d.info( + `Waiting for ${appliedSecrets.length} sealed secrets to be decrypted: ${appliedSecrets.map((s) => s.secretName).join(', ')}`, + ) await retry( async () => { const pending: string[] = [] - for (const secretName of secretNames) { + for (const { namespace, secretName } of appliedSecrets) { try { - const secret = await deps.getK8sSecret(secretName, 'apl-secrets') + const secret = await deps.getK8sSecret(secretName, namespace) if (!secret) { - pending.push(secretName) + pending.push(`${namespace}/${secretName}`) } } catch { - pending.push(secretName) + pending.push(`${namespace}/${secretName}`) } } @@ -172,13 +158,13 @@ export const installAll = async () => { await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') d.info('Applying SealedSecret manifests') - await applySealedSecretManifestsFromDir(env.ENV_DIR) + const appliedSecrets = await applySealedSecretManifestsFromDir(env.ENV_DIR) d.info('Restarting sealed-secrets controller') await restartSealedSecretsController() d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') - await waitForSealedSecrets() + await waitForSealedSecrets(appliedSecrets) // Deploy ESO (External Secrets Operator) d.info('Deploying external-secrets operator') diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index cd764bafda..0d4510d6c8 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -463,24 +463,25 @@ export const applySealedSecretManifests = async ( /** * Read and apply all SealedSecret manifests from the env/manifests/namespaces directory. * This should be called during install, after the sealed-secrets controller is deployed. + * Returns the list of applied secrets (namespace + secretName) so callers can wait for them. */ export const applySealedSecretManifestsFromDir = async ( envDir: string, deps = { terminal, readdir, readFile, existsSync }, -): Promise => { +): Promise<{ namespace: string; secretName: string }[]> => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/namespaces') if (!deps.existsSync(manifestsDir)) { d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) - return + return [] } d.info(`Applying SealedSecret manifests from ${manifestsDir}`) // Read all namespace directories const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) - let appliedCount = 0 + const appliedSecrets: { namespace: string; secretName: string }[] = [] for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue @@ -510,7 +511,7 @@ export const applySealedSecretManifestsFromDir = async ( plural: 'sealedsecrets', body: manifest, }) - appliedCount += 1 + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) } catch (error) { if (error instanceof ApiException && error.code === 409) { try { @@ -525,7 +526,7 @@ export const applySealedSecretManifestsFromDir = async ( }, setHeaderOptions('Content-Type', PatchStrategy.MergePatch), ) - appliedCount += 1 + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) } catch (patchError) { d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) } @@ -539,7 +540,8 @@ export const applySealedSecretManifestsFromDir = async ( } } - d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) + d.info(`Applied ${appliedSecrets.length} SealedSecret manifests from directory`) + return appliedSecrets } /** From 526ee121b7043b34987cfc313b49dfadd63eadda Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 00:23:32 +0100 Subject: [PATCH 45/71] fix: secret data keys --- values/apl-operator/apl-operator-raw.gotmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 9b6652744f..7d1853716a 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -16,8 +16,8 @@ resources: template: type: Opaque data: - GIT_USERNAME: {{ $v.otomi.git | get "username" "otomi-admin" }} - GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' + username: {{ $v.otomi.git | get "username" "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' data: - secretKey: git_password remoteRef: From 22143df9a9e7be9e914863fdcff2bb3438d48034 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:08:54 +0100 Subject: [PATCH 46/71] fix: values-schema x-secret fields --- values-schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/values-schema.yaml b/values-schema.yaml index 45c5dfbafd..584243e4c8 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -907,7 +907,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - default: 'admin' + x-secret: 'admin' securityContext: additionalProperties: uniqueItems: true @@ -1690,6 +1690,7 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: + x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: From b10418ad718212dd16f9c9f11d0925d9ee8ab409 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:57:57 +0100 Subject: [PATCH 47/71] fix: restart sealed secrets controller --- src/cmd/install.ts | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 9bc108f0cc..451feee015 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -61,6 +61,21 @@ const retryInstallStep = async ( * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ +const allSecretsExist = async ( + secrets: { namespace: string; secretName: string }[], + deps = { getK8sSecret }, +): Promise => { + for (const { namespace, secretName } of secrets) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) return false + } catch { + return false + } + } + return true +} + const waitForSealedSecrets = async ( appliedSecrets: { namespace: string; secretName: string }[], timeoutMs = 120000, @@ -160,11 +175,21 @@ export const installAll = async () => { d.info('Applying SealedSecret manifests') const appliedSecrets = await applySealedSecretManifestsFromDir(env.ENV_DIR) - d.info('Restarting sealed-secrets controller') - await restartSealedSecretsController() - - d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') - await waitForSealedSecrets(appliedSecrets) + if (appliedSecrets.length > 0) { + // Check if all secrets are already decrypted (e.g. on retry after a previous successful run) + const allExist = await allSecretsExist(appliedSecrets, { getK8sSecret }) + if (allExist) { + d.info('All sealed secrets already decrypted, skipping controller restart') + } else { + d.info('Restarting sealed-secrets controller to pick up new manifests') + await restartSealedSecretsController() + + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets(appliedSecrets) + } + } else { + d.info('No sealed secret manifests found, skipping controller restart') + } // Deploy ESO (External Secrets Operator) d.info('Deploying external-secrets operator') From 20eb516a4ab93f2b7c01506b1c7328019edbc6e9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:53:14 +0100 Subject: [PATCH 48/71] fix: remove x-secret field from customRootCA --- values-schema.yaml | 1 - values/cert-manager/cert-manager-raw.gotmpl | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/values-schema.yaml b/values-schema.yaml index 584243e4c8..1c0400be78 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -1690,7 +1690,6 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: - x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index e9fc23ce19..af97af4995 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -75,6 +75,7 @@ resources: {{- end }} {{- end }} {{- end }} +{{- if $cm | get "customRootCA" "" }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -97,6 +98,7 @@ resources: remoteRef: key: cert-manager-secrets property: customRootCAKey +{{- end }} - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: From 7f0a422b97d9b2a1ab7b8cd53d9ce18ac62347aa Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 17:27:42 +0100 Subject: [PATCH 49/71] fix: create team settings secrets --- src/common/values.test.ts | 23 +++++++++++++++++++++++ src/common/values.ts | 24 ++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/common/values.test.ts b/src/common/values.test.ts index 06f76fa4e6..3af2dead81 100644 --- a/src/common/values.test.ts +++ b/src/common/values.test.ts @@ -60,6 +60,7 @@ describe('generateSecrets', () => { deps = { terminal, getValuesSchema: jest.fn().mockReturnValue(schema), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), } }) it('should generate new secrets and return only secrets', async () => { @@ -72,4 +73,26 @@ describe('generateSecrets', () => { const res = await generateSecrets(valuesWithExisting, deps) expect(res.nested.twoStage).toBe('exists') }) + it('should include team secrets with expanded paths', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.demo.settings.password', 'team-secret-pw') + + deps.getSchemaSecretsPaths.mockResolvedValue(['teamConfig.demo.settings.password']) + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledWith(['demo']) + expect(res.teamConfig.demo.settings.password).toBe('team-secret-pw') + }) + it('should not call getSchemaSecretsPaths when no dynamic teams exist', async () => { + const res = await generateSecrets(values, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + expect(res).toEqual(expected) + }) + it('should exclude admin team from dynamic team expansion', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.admin.settings.password', 'admin-pw') + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + }) }) diff --git a/src/common/values.ts b/src/common/values.ts index de44be2dbf..e8270d034a 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' +import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -10,7 +10,16 @@ import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' import { saveValues } from './repo' -import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' +import { + extract, + flattenObject, + getSchemaSecretsPaths, + getValuesSchema, + gucci, + loadYaml, + pkg, + removeBlankAttributes, +} from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -135,6 +144,7 @@ export const generateSecrets = async ( deps = { terminal, getValuesSchema, + getSchemaSecretsPaths, }, ): Promise> => { const d = deps.terminal('common:values:generateSecrets') @@ -158,6 +168,16 @@ export const generateSecrets = async ( // Only return values that have x-secrets prop and are now fully templated: const templatePaths = Object.keys(flattenObject(schemaSecrets)) const res = pick(allSecrets, templatePaths) + + // Template paths use schema patternProperties regex keys which don't match concrete team names. + // Expand team paths so team secrets are included in the result. + const teamNames = Object.keys(get(values, 'teamConfig', {})).filter((t) => t !== 'admin') + if (teamNames.length > 0) { + const expandedPaths = await deps.getSchemaSecretsPaths(teamNames) + const teamSecrets = pick(allSecrets, expandedPaths) + merge(res, teamSecrets) + } + d.debug('generateSecrets result: ', res) return res } From f333c235b0c8bf15ed07818887cdf5c69c8d06ca Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:33:04 +0100 Subject: [PATCH 50/71] fix: harbor push issues --- src/cmd/pull.ts | 2 +- values-schema.yaml | 2 +- values/k8s/k8s-raw.gotmpl | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 550773d67a..28b7f6edb7 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -17,7 +17,7 @@ export const pull = async (): Promise => { d.error('No values found, skipping git pull') return } - const gitRepo = getRepo(allValues) + const gitRepo = await getRepo(allValues) const { branch } = gitRepo d.info('Pulling latest values') cd(env.ENV_DIR) diff --git a/values-schema.yaml b/values-schema.yaml index 1c0400be78..f1b0c184fa 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -907,7 +907,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - x-secret: 'admin' + x-secret: 'otomi-admin' securityContext: additionalProperties: uniqueItems: true diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 65631e6f26..1eec03a63e 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -72,3 +72,16 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." + {{- if $v.cluster.domainSuffix }} + # CoreDNS custom config to resolve platform domains to the ingress controller ClusterIP. + # This avoids hairpin NAT issues where pods cannot reach services via the external LoadBalancer IP. + - apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns-custom + namespace: kube-system + data: + otomi-hairpin.include: | + {{- $escapedDomain := $v.cluster.domainSuffix | replace "." "\\." }} + rewrite name regex (.+)\.{{ $escapedDomain }} ingress-nginx-platform-controller.ingress.svc.cluster.local answer auto + {{- end }} From f49a5db3613238c69b9b7070cbd2ba866fe6f3e0 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:27:49 +0100 Subject: [PATCH 51/71] feat: use commands with cwd instead of cd --- src/cmd/commit.ts | 42 +++++++++++++++++++---------------------- src/cmd/pull.ts | 8 ++++---- src/common/bootstrap.ts | 7 ++++--- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 89faf6e757..00f381848d 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -12,12 +12,12 @@ import { createUpdateConfigMap, createUpdateGenericSecret, getK8sSecret, k8s } f import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -export const rootDir = process.cwd() === '/home/app/stack/env' ? '/home/app/stack' : process.cwd() +const $git = $({ cwd: env.ENV_DIR }) interface Arguments extends HelmArguments { m?: string @@ -43,12 +43,11 @@ const isConflictError = (error: any): boolean => { const cleanupGitState = async (d: any): Promise => { try { - cd(env.ENV_DIR) // Try to abort any ongoing merge or rebase - await $`git merge --abort`.nothrow().quiet() - await $`git rebase --abort`.nothrow().quiet() + await $git`git merge --abort`.nothrow().quiet() + await $git`git rebase --abort`.nothrow().quiet() // Reset to the commit before our failed commit to discard local changes - await $`git reset --hard HEAD~1`.quiet() + await $git`git reset --hard HEAD~1`.quiet() d.info('Git state cleaned up after conflict - local commit discarded, reconciliation will retry') } catch (cleanupError) { d.warn('Error during git cleanup:', cleanupError?.message) @@ -65,25 +64,25 @@ const commitAndPush = async ( d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' const { password } = gitConfig ?? (await getRepo(values)) - cd(env.ENV_DIR) try { try { - await $`git rev-list HEAD --count`.quiet() + await $git`git rev-list HEAD --count`.quiet() } catch { d.log('Very first commit') // We need at least two commits in repo, so git diff in Tekton pipeline always works. This is why the very first time we commit twice. - await $`git add README.md`.quiet() - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git add README.md`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } - await $`git add -A` + await $git`git add -A` // The below 'git status' command will always return at least single new line - const filesChangedCount = (await $`git status --untracked-files=no --porcelain`).toString().split('\n').length - 1 + const statusOutput = (await $git`git status --untracked-files=no --porcelain`).toString() + const filesChangedCount = statusOutput.split('\n').length - 1 if (filesChangedCount === 0) { d.log('Nothing to commit') return } - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } catch (e) { const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` d.error(errorMsg) @@ -93,24 +92,23 @@ const commitAndPush = async ( await retry( async () => { try { - cd(env.ENV_DIR) // Check if remote branch exists let remoteBranchExists = true try { - await $`git ls-remote --exit-code --heads origin ${branch}`.quiet() + await $git`git ls-remote --exit-code --heads origin ${branch}`.quiet() } catch { remoteBranchExists = false } // We're not always sure that we are on the correct branch, // so we checkout the branch and create it if it does not exist - await $`git checkout -B ${branch}`.quiet() + await $git`git checkout -B ${branch}`.quiet() if (remoteBranchExists) { - await $`git pull --rebase origin ${branch}`.quiet() + await $git`git pull --rebase origin ${branch}`.quiet() } else { d.log(`Remote branch '${branch}' does not exist. Skipping pull.`) } - await $`git push -u origin ${branch}`.quiet() + await $git`git push -u origin ${branch}`.quiet() } catch (pullPushError) { // Check if this is a merge conflict - if so, skip the commit if (isConflictError(pullPushError)) { @@ -152,13 +150,11 @@ export const commit = async ( // the bootstrap phase before install) may have set the URL with unresolved placeholder // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, // we need to update the URL with the real credentials. - cd(env.ENV_DIR) - await $`git remote set-url origin ${remote}`.nothrow().quiet() + await $git`git remote set-url origin ${remote}`.nothrow().quiet() } else { - cd(env.ENV_DIR) - await setIdentity(username, email) + await setIdentity(username, email, env.ENV_DIR) // the url might need updating (e.g. if credentials changed) - await $`git remote set-url origin ${remote}` + await $git`git remote set-url origin ${remote}` } // let's wait until the remote is ready if (values?.apps!.gitea!.enabled ?? true) { diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 28b7f6edb7..b56ab13262 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -5,7 +5,7 @@ import { hfValues } from 'src/common/hf' import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { getRepo } from '../common/git-config' const cmdName = getFilename(__filename) @@ -20,10 +20,10 @@ export const pull = async (): Promise => { const gitRepo = await getRepo(allValues) const { branch } = gitRepo d.info('Pulling latest values') - cd(env.ENV_DIR) + const $git = $({ cwd: env.ENV_DIR }) try { - await $`git fetch` - await $`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` + await $git`git fetch` + await $git`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` } catch (error) { d.warn( `An error occured when trying to pull (maybe not problematic).\nIf you see merge conflicts then please resolve these and run \`otomi commit\` again.`, diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index fe82d245a6..587a5a4031 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -12,9 +12,10 @@ import { $, cd } from 'zx' const cmdName = getFilename(__filename) -export const setIdentity = async (username, email) => { - await $`git config --local user.name ${username}`.nothrow().quiet() - await $`git config --local user.email ${email}`.nothrow().quiet() +export const setIdentity = async (username, email, cwd?: string) => { + const $run = cwd ? $({ cwd }) : $ + await $run`git config --local user.name ${username}`.nothrow().quiet() + await $run`git config --local user.email ${email}`.nothrow().quiet() } export const recoverFromGit = async (gitConfig: GitRepoConfig): Promise => { From db4524b1e982ada6b2882c9a954011777359537d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 21:14:45 +0100 Subject: [PATCH 52/71] fix: use commands with cwd instead of cd --- src/common/gitea.ts | 8 ++++---- src/common/hf.ts | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common/gitea.ts b/src/common/gitea.ts index ab6268419b..5d46cbdafd 100644 --- a/src/common/gitea.ts +++ b/src/common/gitea.ts @@ -1,5 +1,5 @@ import retry from 'async-retry' -import { $, cd } from 'zx' +import { $ } from 'zx' import { APL_OPERATOR_NS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -29,14 +29,14 @@ export const waitTillGitRepoAvailable = async (repoUrl: string): Promise = const d = terminal('common:gitea:waitTillGitRepoAvailable') await retry( async () => { + const $git = $({ cwd: env.ENV_DIR }) try { - cd(env.ENV_DIR) // the ls-remote exists with zero even if repo is empty - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } catch (e) { if (e.stderr && e.stderr.includes('remote: Update your password')) { await resetGiteaPasswordValidity() - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } else { d.warn(`The values repository is not yet reachable. Retrying in ${env.MIN_TIMEOUT} ms`) throw e diff --git a/src/common/hf.ts b/src/common/hf.ts index 53f41bf450..4068fef4bc 100644 --- a/src/common/hf.ts +++ b/src/common/hf.ts @@ -64,10 +64,12 @@ const hfCore = (args: HFParams, envDir = env.ENV_DIR): ProcessPromise => { stringArray.push(`--log-level=${paramsCopy.logLevel.toLowerCase()}`) process.env.HELM_DIFF_COLOR = 'true' process.env.HELM_DIFF_USE_UPGRADE_DRY_RUN = 'true' + // Always run helmfile from rootDir to ensure helmfile.d/ is found regardless of process cwd + const $hf = $({ cwd: rootDir }) if ((parsedArgs?.dryRun || parsedArgs?.local) && paramsCopy.args.includes('sync')) { - return $`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } else { - return $`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } } From dd8419efe8233f1cbda4be6e67b7239eb4e9bd1f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:13:09 +0100 Subject: [PATCH 53/71] fix: update sealed secrets handling and improve deployment configurations --- chart/apl/templates/sops-secrets.yaml | 35 ------ chart/chart-index/Chart.yaml | 3 + charts/apl-operator/templates/deployment.yaml | 1 - .../ingress-nginx/templates/clusterrole.yaml | 1 - src/cmd/commit.ts | 12 +- src/cmd/install.ts | 18 +-- src/cmd/migrate.ts | 11 +- src/common/constants.ts | 3 + src/common/git-config.ts | 4 +- src/common/sealed-secrets.test.ts | 6 +- src/common/sealed-secrets.ts | 114 +++++++----------- 11 files changed, 81 insertions(+), 127 deletions(-) delete mode 100644 chart/apl/templates/sops-secrets.yaml diff --git a/chart/apl/templates/sops-secrets.yaml b/chart/apl/templates/sops-secrets.yaml deleted file mode 100644 index 7544e0ec6c..0000000000 --- a/chart/apl/templates/sops-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- $kms := .Values.kms | default dict }} -{{- if hasKey $kms "sops" }} -{{- $v := $kms.sops }} -apiVersion: v1 -kind: Secret -metadata: - name: apl-sops-secrets - namespace: apl-operator -type: Opaque -data: -{{- with $v.azure }} - AZURE_CLIENT_ID: {{ .clientId | b64enc }} - AZURE_CLIENT_SECRET: {{ .clientSecret | b64enc }} -{{- with .tenantId }} - AZURE_TENANT_ID: {{ . | b64enc }}{{ end }} -{{- with .environment }} - AZURE_ENVIRONMENT: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.aws }} - AWS_ACCESS_KEY_ID: {{ .accessKey | b64enc }} - AWS_SECRET_ACCESS_KEY: {{ .secretKey | b64enc }} -{{- with .region }} - AWS_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.age }} - SOPS_AGE_KEY: {{ .privateKey | b64enc }} -{{- end }} -{{- with $v.google }} - GCLOUD_SERVICE_KEY: {{ .accountJson | b64enc }} -{{- with .project }} - GOOGLE_PROJECT: {{ . | b64enc }}{{ end }} -{{- with .region }} - GOOGLE_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/chart/chart-index/Chart.yaml b/chart/chart-index/Chart.yaml index 908ec6f1d0..c4eea630db 100644 --- a/chart/chart-index/Chart.yaml +++ b/chart/chart-index/Chart.yaml @@ -31,6 +31,9 @@ dependencies: - name: external-dns version: 1.20.0 repository: https://kubernetes-sigs.github.io/external-dns + - name: external-secrets + version: 0.14.3 + repository: https://charts.external-secrets.io - name: gitea version: 12.5.0 repository: https://dl.gitea.io/charts diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index 5f28093cc1..0991e2e09b 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -47,7 +47,6 @@ spec: optional: true - secretRef: name: apl-git-credentials - optional: true livenessProbe: exec: command: diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 15b2ac4b55..51bc5002cc 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,7 +27,6 @@ rules: - namespaces {{- end}} verbs: - - get - list - watch - apiGroups: diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 00f381848d..0b8b5e08b3 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,7 +1,13 @@ import retry from 'async-retry' import { bootstrapGit, setIdentity } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' -import { APL_OPERATOR_NS, DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' +import { + APL_OPERATOR_NS, + DEPLOYMENT_PASSWORDS_SECRET, + OTOMI_NAMESPACE, + OTOMI_PLATFORM_SECRETS, + SEALED_SECRETS_NAMESPACE, +} from 'src/common/constants' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -176,7 +182,7 @@ export async function initialSetupData(): Promise { // Read the platform admin's initialPassword from the generated passwords secret let platformAdminPassword = '' try { - const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi') + const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, OTOMI_NAMESPACE) const allSecrets = secretData?.[DEPLOYMENT_PASSWORDS_SECRET] const users = allSecrets?.users || [] const defaultEmail = `platform-admin@${domainSuffix}` @@ -193,7 +199,7 @@ export async function initialSetupData(): Promise { } } else { // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const otomiSecret = await getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 451feee015..a15ab01ef5 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -17,7 +17,11 @@ import { setDeploymentState, waitForCRD, } from 'src/common/k8s' -import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' +import { + AppliedSecret, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -61,10 +65,7 @@ const retryInstallStep = async ( * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ -const allSecretsExist = async ( - secrets: { namespace: string; secretName: string }[], - deps = { getK8sSecret }, -): Promise => { +const allSecretsExist = async (secrets: AppliedSecret[], deps = { getK8sSecret }): Promise => { for (const { namespace, secretName } of secrets) { try { const secret = await deps.getK8sSecret(secretName, namespace) @@ -77,7 +78,7 @@ const allSecretsExist = async ( } const waitForSealedSecrets = async ( - appliedSecrets: { namespace: string; secretName: string }[], + appliedSecrets: AppliedSecret[], timeoutMs = 120000, intervalMs = 3000, deps = { getK8sSecret, terminal }, @@ -108,7 +109,6 @@ const waitForSealedSecrets = async ( } if (pending.length > 0) { - d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) throw new Error(`Sealed secrets not yet decrypted: ${pending.join(', ')}`) } @@ -181,7 +181,9 @@ export const installAll = async () => { if (allExist) { d.info('All sealed secrets already decrypted, skipping controller restart') } else { - d.info('Restarting sealed-secrets controller to pick up new manifests') + // The controller may have started before the sealed-secrets-key TLS secret existed, + // causing it to generate its own key. Restarting forces it to pick up the pre-created key. + d.info('Restarting sealed-secrets controller to ensure correct key is used') await restartSealedSecretsController() d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 830ca62fd4..46e5952f33 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,7 +20,12 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' -import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS } from '../common/constants' +import { + APL_OPERATOR_NS, + ARGOCD_APP_PARAMS, + OTOMI_PLATFORM_SECRETS, + SEALED_SECRETS_NAMESPACE, +} from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' import { applySealedSecretManifestsFromDir, @@ -725,7 +730,7 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - const giteaSecrets = await getK8sSecret('gitea-secrets', 'apl-secrets') + const giteaSecrets = await getK8sSecret('gitea-secrets', SEALED_SECRETS_NAMESPACE) const resolvedGitea = { adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), @@ -895,7 +900,7 @@ export const sopsMigration = async ( }) if (existingManifests.length > 0) { try { - const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const platformSecret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) if (!platformSecret) { d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) diff --git a/src/common/constants.ts b/src/common/constants.ts index 283c14d69a..bb67c8919f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -2,6 +2,9 @@ export const DEPLOYMENT_PASSWORDS_SECRET = 'otomi-generated-passwords' export const DEPLOYMENT_STATUS_CONFIGMAP = 'otomi-status' export const APL_OPERATOR_NS = 'apl-operator' export const APL_OPERATOR_STATUS_CM = 'apl-installation-status' +export const OTOMI_NAMESPACE = 'otomi' +export const SEALED_SECRETS_NAMESPACE = 'apl-secrets' +export const OTOMI_PLATFORM_SECRETS = 'otomi-platform-secrets' export const ARGOCD_APP_PARAMS = { group: 'argoproj.io', version: 'v1alpha1', diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 9498ee6fe5..662df6fb0b 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,5 +1,5 @@ import type { CoreV1Api } from '@kubernetes/client-node' -import { APL_OPERATOR_NS } from './constants' +import { APL_OPERATOR_NS, OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' @@ -160,7 +160,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const secret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 25835a38a9..8874db2784 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -10,8 +10,8 @@ import { generateSealedSecretsKeyPair, getPemFromCertificate, restartSealedSecretsController, - SECRET_NAME_MAP, SealedSecretManifest, + SECRET_NAME_MAP, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -572,12 +572,12 @@ describe('sealed-secrets', () => { await applySealedSecretManifests(manifests, { terminal }) const { k8s: mockK8s } = require('src/common/k8s') - expect(mockK8s.custom().createNamespacedCustomObject).toHaveBeenCalled() + expect(mockK8s.custom().patchNamespacedCustomObject).toHaveBeenCalled() }) it('should log error when manifests fail to apply', async () => { const { k8s: mockK8s } = require('src/common/k8s') - mockK8s.custom().createNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) + mockK8s.custom().patchNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) const manifests = [makeMockManifest('test-secret', 'apl-secrets')] diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 0d4510d6c8..b04882892b 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -6,6 +6,7 @@ import { mkdir, readdir, readFile, writeFile } from 'fs/promises' import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' +import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' import { terminal } from 'src/common/debug' import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' @@ -13,6 +14,8 @@ import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' +const ROLLOUT_TIMEOUT_MS = 120000 +const ROLLOUT_INTERVAL_MS = 3000 /** * Strip ALL x-secret fields from values before writing to disk. @@ -51,17 +54,21 @@ export interface SealedSecretManifest { } } -/** - * All SealedSecrets are placed in the 'apl-secrets' namespace. - * ESO ClusterSecretStore reads from this namespace and distributes secrets to target namespaces. - */ -const SEALED_SECRETS_NAMESPACE = 'apl-secrets' +export interface SealedSecretsKeyPair { + certificate: string + privateKey: string +} + +export interface AppliedSecret { + namespace: string + secretName: string +} /** * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. * Follows the pattern from createCustomCA() in bootstrap.ts. */ -export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): SealedSecretsKeyPair => { const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) d.info('Generating sealed-secrets RSA key pair') @@ -399,6 +406,26 @@ export const writeSealedSecretManifests = async ( } } +/** + * Apply a single SealedSecret manifest using server-side apply (create-or-update). + * Uses the same pattern as applyArgocdApp() in apply-as-apps.ts. + */ +const applySealedSecretResource = async (manifest: SealedSecretManifest): Promise => { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace: manifest.metadata.namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + fieldManager: 'apl-operator', + force: true, + }, + setHeaderOptions('Content-Type', PatchStrategy.ServerSideApply), + ) +} + /** * Apply SealedSecret manifests to the Kubernetes cluster. * Creates namespaces if needed and applies the SealedSecret resources. @@ -426,33 +453,9 @@ export const applySealedSecretManifests = async ( for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) try { - await k8s.custom().createNamespacedCustomObject({ - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - body: manifest, - }) + await applySealedSecretResource(manifest) } catch (error) { - if (error instanceof ApiException && error.code === 409) { - try { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - } catch (patchError) { - d.error(`Failed to patch SealedSecret ${manifest.metadata.name}: ${patchError}`) - } - } else { - d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) - } + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) } } } @@ -468,7 +471,7 @@ export const applySealedSecretManifests = async ( export const applySealedSecretManifestsFromDir = async ( envDir: string, deps = { terminal, readdir, readFile, existsSync }, -): Promise<{ namespace: string; secretName: string }[]> => { +): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/namespaces') @@ -481,7 +484,7 @@ export const applySealedSecretManifestsFromDir = async ( // Read all namespace directories const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) - const appliedSecrets: { namespace: string; secretName: string }[] = [] + const appliedSecrets: AppliedSecret[] = [] for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue @@ -503,39 +506,10 @@ export const applySealedSecretManifestsFromDir = async ( const content = await deps.readFile(filePath, 'utf-8') const manifest = parseYaml(content) as SealedSecretManifest - try { - await k8s.custom().createNamespacedCustomObject({ - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - body: manifest, - }) - appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) - } catch (error) { - if (error instanceof ApiException && error.code === 409) { - try { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) - } catch (patchError) { - d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) - } - } else { - d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) - } - } - } catch (parseError) { - d.error(`Failed to parse SealedSecret from ${filePath}: ${parseError}`) + await applySealedSecretResource(manifest) + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) + } catch (error) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) } } } @@ -578,10 +552,8 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi } d.info('Waiting for sealed-secrets controller rollout') - const timeoutMs = 120000 - const intervalMs = 3000 const start = Date.now() - while (Date.now() - start < timeoutMs) { + while (Date.now() - start < ROLLOUT_TIMEOUT_MS) { try { const deployment = await k8s.app().readNamespacedDeployment({ name: 'sealed-secrets', @@ -597,7 +569,7 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi } catch { // Ignore transient read errors during rollout } - await new Promise((resolve) => setTimeout(resolve, intervalMs)) + await new Promise((resolve) => setTimeout(resolve, ROLLOUT_INTERVAL_MS)) } d.warn('Rollout status check timed out') } From ace06e202d50fd783de213623cb3ea4b7bc2183b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:13:52 +0100 Subject: [PATCH 54/71] fix: update sealed secrets handling and rename secrets --- src/cmd/commit.ts | 9 +- src/cmd/migrate.test.ts | 4 +- src/cmd/migrate.ts | 9 +- src/common/constants.ts | 4 +- src/common/git-config.test.ts | 6 +- src/common/git-config.ts | 4 +- src/common/sealed-secrets.test.ts | 29 ++++-- src/common/sealed-secrets.ts | 97 ++++--------------- src/operator/installer.test.ts | 3 +- src/operator/installer.ts | 6 +- src/operator/main.ts | 6 -- .../apl-gitea-operator-raw.gotmpl | 2 +- .../apl-harbor-operator-raw.gotmpl | 2 +- .../apl-keycloak-operator-raw.gotmpl | 2 +- values/apl-operator/apl-operator-raw.gotmpl | 2 +- values/argocd/argocd-raw.gotmpl | 6 +- values/gitea/gitea-raw.gotmpl | 4 +- values/harbor/harbor-raw.gotmpl | 6 +- values/keycloak/keycloak-raw.gotmpl | 4 +- values/loki/loki-raw.gotmpl | 2 +- values/otomi-api/otomi-api-raw.gotmpl | 2 +- .../prometheus-operator-raw.gotmpl | 2 +- 22 files changed, 77 insertions(+), 134 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 0b8b5e08b3..4a4d9417a7 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -5,7 +5,7 @@ import { APL_OPERATOR_NS, DEPLOYMENT_PASSWORDS_SECRET, OTOMI_NAMESPACE, - OTOMI_PLATFORM_SECRETS, + OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE, } from 'src/common/constants' import { encrypt } from 'src/common/crypt' @@ -198,12 +198,13 @@ export async function initialSetupData(): Promise { secretName, } } else { - // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + // External IDP: show Keycloak admin credentials + const adminUsername = values?.apps?.keycloak?.adminUsername || 'otomi-admin' + const otomiSecret = await getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, - username: 'otomi-admin', + username: adminUsername, password: adminPassword, secretName, } diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 26cfcf0b26..e2f9a895fb 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -969,7 +969,7 @@ describe('sopsMigration', () => { it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) @@ -981,7 +981,7 @@ describe('sopsMigration', () => { it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 46e5952f33..c49b85841b 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,12 +20,7 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' -import { - APL_OPERATOR_NS, - ARGOCD_APP_PARAMS, - OTOMI_PLATFORM_SECRETS, - SEALED_SECRETS_NAMESPACE, -} from '../common/constants' +import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' import { applySealedSecretManifestsFromDir, @@ -900,7 +895,7 @@ export const sopsMigration = async ( }) if (existingManifests.length > 0) { try { - const platformSecret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + const platformSecret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) if (!platformSecret) { d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) diff --git a/src/common/constants.ts b/src/common/constants.ts index bb67c8919f..b48480b27f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -4,7 +4,7 @@ export const APL_OPERATOR_NS = 'apl-operator' export const APL_OPERATOR_STATUS_CM = 'apl-installation-status' export const OTOMI_NAMESPACE = 'otomi' export const SEALED_SECRETS_NAMESPACE = 'apl-secrets' -export const OTOMI_PLATFORM_SECRETS = 'otomi-platform-secrets' +export const OTOMI_SECRETS = 'otomi-secrets' export const ARGOCD_APP_PARAMS = { group: 'argoproj.io', version: 'v1alpha1', @@ -17,7 +17,7 @@ export const ARGOCD_APP_DEFAULT_SYNC_POLICY = { allowEmpty: false, selfHeal: true, }, - syncOptions: ['ServerSideApply=true', 'CreateNamespace=true'], + syncOptions: ['ServerSideApply=true'], } export interface ObjectMetadata { diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index ef0c5a2ab2..c68a6a1c37 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -71,7 +71,7 @@ describe('git-config', () => { it('should return undefined when password is a sealed-secret placeholder', async () => { mockGetK8sSecret.mockResolvedValue({ username: 'admin', - password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-secrets/git_password', }) const result = await getGitCredentials() expect(result).toBeUndefined() @@ -362,7 +362,7 @@ describe('git-config', () => { git: { repoUrl: 'https://github.com/org/repo.git', username: 'admin', - password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-secrets/git_password', branch: 'main', email: 'pipeline@cluster.local', }, @@ -370,7 +370,7 @@ describe('git-config', () => { } const result = await getRepo(values, { getK8sSecret: secretMock }) - expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'apl-secrets') + expect(secretMock).toHaveBeenCalledWith('otomi-secrets', 'apl-secrets') expect(result.password).toBe('real-password') expect(result.authenticatedUrl).toContain('real-password') }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 662df6fb0b..bf15462522 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,5 +1,5 @@ import type { CoreV1Api } from '@kubernetes/client-node' -import { APL_OPERATOR_NS, OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' +import { APL_OPERATOR_NS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' @@ -160,7 +160,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 8874db2784..cee5b3ab0e 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -11,7 +11,6 @@ import { getPemFromCertificate, restartSealedSecretsController, SealedSecretManifest, - SECRET_NAME_MAP, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -502,14 +501,26 @@ describe('sealed-secrets', () => { }) }) - describe('SECRET_NAME_MAP', () => { - it('should have expected secret name mappings', () => { - expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') - expect(SECRET_NAME_MAP['apps.gitea']).toBe('gitea-secrets') - expect(SECRET_NAME_MAP['apps.keycloak']).toBe('keycloak-secrets') - expect(SECRET_NAME_MAP['otomi']).toBe('otomi-platform-secrets') - expect(SECRET_NAME_MAP['oidc']).toBe('oidc-secrets') - expect(SECRET_NAME_MAP['dns']).toBe('dns-secrets') + describe('secret name derivation', () => { + it('should derive correct secret names via buildSecretToNamespaceMap', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + otomi: { adminPassword: 'pass' }, + obj: { provider: { linode: { secretAccessKey: 'key' } } }, + dns: { provider: { linode: { apiToken: 'token' } } }, + } + const result = await buildSecretToNamespaceMap(secrets, [], undefined, { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.harbor.adminPassword', + 'otomi.adminPassword', + 'obj.provider.linode.secretAccessKey', + 'dns.provider.linode.apiToken', + ]), + }) + const names = result.map((m) => m.secretName).sort() + expect(names).toEqual(['dns-secrets', 'harbor-secrets', 'obj-secrets', 'otomi-secrets']) }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index b04882892b..1f92b9e4e1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -205,94 +205,40 @@ export const createSealedSecretsKeySecret = async ( } } -/** - * Resolve the namespace for a given secret path. - * All secrets go to 'apl-secrets' namespace for ESO ClusterSecretStore access. - */ -const resolveNamespace = (secretPath: string): string | undefined => { - // Check for teamConfig dynamic paths - if (secretPath.match(/^teamConfig\.[^.]+/)) { - return SEALED_SECRETS_NAMESPACE - } - - // Check if this path matches any known secret name prefix - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return SEALED_SECRETS_NAMESPACE - } - } - - return undefined -} - -// Map specific path prefixes to secret names -export const SECRET_NAME_MAP: Record = { - 'apps.harbor': 'harbor-secrets', - 'apps.gitea': 'gitea-secrets', - 'apps.keycloak': 'keycloak-secrets', - 'apps.grafana': 'grafana-secrets', - 'apps.loki': 'loki-secrets', - 'apps.oauth2-proxy': 'oauth2-proxy-secrets', - 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', - 'apps.prometheus': 'prometheus-secrets', - 'apps.otomi-api': 'otomi-api-secrets', - 'apps.cert-manager': 'cert-manager-secrets', - 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', - otomi: 'otomi-platform-secrets', - oidc: 'oidc-secrets', - smtp: 'smtp-secrets', - dns: 'dns-secrets', - obj: 'obj-storage-secrets', - license: 'license-secrets', - alerts: 'alerts-secrets', - cluster: 'cluster-secrets', -} - /** * Find the group prefix for a secret path. - * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + * Groups: teamConfig.X, apps.X, or a single top-level key (e.g., 'otomi', 'dns'). */ const findGroupPrefix = (secretPath: string): string | undefined => { const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return `teamConfig.${teamMatch[1]}` - } + if (teamMatch) return `teamConfig.${teamMatch[1]}` - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return prefix - } - } + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `apps.${appsMatch[1]}` + + // Top-level paths: use the first segment as the group prefix + const [firstSegment] = secretPath.split('.') + // Skip paths like 'kms' and 'users' which are handled separately + if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment - // Fallback: use first two path segments - const parts = secretPath.split('.') - if (parts.length >= 2) { - return parts.slice(0, 2).join('.') - } return undefined } /** - * Derive a K8s secret name from the secret path prefix. + * Derive a K8s secret name from a secret path. + * Convention: all secrets follow {name}-secrets pattern. + * - teamConfig.X -> team-X-settings-secrets + * - apps.X -> X-secrets + * - topLevel -> topLevel-secrets */ const deriveSecretName = (secretPath: string): string => { const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return `team-${teamMatch[1]}-settings-secrets` - } + if (teamMatch) return `team-${teamMatch[1]}-settings-secrets` - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return SECRET_NAME_MAP[prefix] - } - } + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `${appsMatch[1]}-secrets` - // Fallback: derive from first two path segments - const parts = secretPath.split('.') - return `${parts.slice(0, 2).join('-')}-secrets` + return `${secretPath.split('.')[0]}-secrets` } /** @@ -317,14 +263,13 @@ export const buildSecretToNamespaceMap = async ( // Skip users path — user secrets are managed individually in apl-users namespace if (secretPath === 'users') continue - const namespace = resolveNamespace(secretPath) - if (!namespace) continue + if (!findGroupPrefix(secretPath)) continue const secretName = deriveSecretName(secretPath) - const groupKey = `${namespace}/${secretName}` + const groupKey = `${SEALED_SECRETS_NAMESPACE}/${secretName}` if (!groupMap.has(groupKey)) { - groupMap.set(groupKey, { namespace, secretName, data: {} }) + groupMap.set(groupKey, { namespace: SEALED_SECRETS_NAMESPACE, secretName, data: {} }) } const mapping = groupMap.get(groupKey)! diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index e12f58e6d6..400d069f62 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -169,7 +169,7 @@ describe('Installer', () => { }), ) - // Verify failed status was recorded with error message + // Verify failed status was recorded expect(k8s.createUpdateConfigMap).toHaveBeenCalledWith( mockCoreApi, 'apl-installation-status', @@ -177,7 +177,6 @@ describe('Installer', () => { expect.objectContaining({ status: 'failed', attempt: '1', - error: 'Install failed', }), ) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 03835a3faf..8deb485858 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -170,7 +170,7 @@ export class Installer { } catch (error) { const errorMessage = getErrorMessage(error) this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) - await this.updateInstallationStatus('failed', attemptNumber, errorMessage) + await this.updateInstallationStatus('failed', attemptNumber) // Clean up stuck Helm releases (e.g. pending-install, pending-upgrade) // so the next retry can proceed without "another operation is in progress" errors @@ -194,14 +194,12 @@ export class Installer { return status } - private async updateInstallationStatus(status: string, attempt: number, error?: string): Promise { + private async updateInstallationStatus(status: string, attempt: number): Promise { try { const data = { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), - // Always include error field to prevent stale values from StrategicMergePatch - error: error ?? '', } await createUpdateConfigMap(k8s.core(), APL_OPERATOR_STATUS_CM, APL_OPERATOR_NS, data) diff --git a/src/operator/main.ts b/src/operator/main.ts index 38d45eb407..26c50691e4 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -2,7 +2,6 @@ import * as dotenv from 'dotenv' import fs from 'fs' import process from 'node:process' import path from 'path' -import { runTraceCollectionLoop } from '../cmd/traces' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { getStoredGitRepoConfig } from '../common/git-config' @@ -93,11 +92,6 @@ async function main(): Promise { // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) await installer.setEnvAndCreateSecrets() - // Start trace collection in background (runs for 30 minutes from ConfigMap creation) - runTraceCollectionLoop().catch((error) => { - d.warn('Trace collection loop failed:', getErrorMessage(error)) - }) - // Phase 2: Set environment variables and start operator for GitOps operations const config = await loadConfig(aplOps) const operator = new AplOperator(config) diff --git a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl index a89537c3ca..d6b6c4c167 100644 --- a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl +++ b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl @@ -42,7 +42,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password - secretKey: keycloakClientSecret remoteRef: diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 0eaf85f5a3..265e5058e0 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -35,7 +35,7 @@ resources: data: - secretKey: harborAdminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - secretKey: keycloakClientSecret remoteRef: diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 43763a08fa..b07da67e9b 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -48,7 +48,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - secretKey: idpClientSecret remoteRef: diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 7d1853716a..41228f7257 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -21,5 +21,5 @@ resources: data: - secretKey: git_password remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 8c9966a6f9..d400ea4a16 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -36,7 +36,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -64,7 +64,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- else }} - apiVersion: external-secrets.io/v1beta1 @@ -93,7 +93,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- end }} - apiVersion: external-secrets.io/v1beta1 diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 2098889f8d..d7d84c04a4 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -33,7 +33,7 @@ resources: data: - secretKey: git_password remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- if $v._derived.untrustedCA }} - apiVersion: v1 @@ -65,7 +65,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} {{- with $v | get "smtp" nil }} diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index fde1bcc6c9..4f1f61ffe6 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -52,7 +52,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -206,7 +206,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -228,6 +228,6 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} diff --git a/values/keycloak/keycloak-raw.gotmpl b/values/keycloak/keycloak-raw.gotmpl index dcf7214cc4..ef85209a5b 100644 --- a/values/keycloak/keycloak-raw.gotmpl +++ b/values/keycloak/keycloak-raw.gotmpl @@ -30,7 +30,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword {{- if eq $obj.type "linode" }} - apiVersion: external-secrets.io/v1beta1 @@ -53,6 +53,6 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} diff --git a/values/loki/loki-raw.gotmpl b/values/loki/loki-raw.gotmpl index 3ca9d69809..bbebacc435 100644 --- a/values/loki/loki-raw.gotmpl +++ b/values/loki/loki-raw.gotmpl @@ -66,7 +66,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} {{- end }} diff --git a/values/otomi-api/otomi-api-raw.gotmpl b/values/otomi-api/otomi-api-raw.gotmpl index 51e4f6023b..a3169981dc 100644 --- a/values/otomi-api/otomi-api-raw.gotmpl +++ b/values/otomi-api/otomi-api-raw.gotmpl @@ -20,5 +20,5 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index b6ef5709b9..0805b2c827 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -29,7 +29,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret From c7dc6e3e2733790ec7cfb38cc5167695d2f26ede Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 14:36:22 +0100 Subject: [PATCH 55/71] feat: remove deprecated secret files from tests/fixtures and replace them with SealedSecrets --- src/common/repo.test.ts | 68 +-------- src/common/repo.ts | 136 +++++++++++++----- tests/fixtures/env/apps/cert-manager.yaml | 22 +++ .../env/apps/secrets.cert-manager.yaml | 55 ------- tests/fixtures/env/apps/secrets.gitea.yaml | 6 - tests/fixtures/env/apps/secrets.grafana.yaml | 6 - tests/fixtures/env/apps/secrets.harbor.yaml | 18 --- tests/fixtures/env/apps/secrets.keycloak.yaml | 7 - .../env/apps/secrets.kubeflow-pipelines.yaml | 6 - tests/fixtures/env/apps/secrets.loki.yaml | 6 - .../env/apps/secrets.oauth2-proxy-redis.yaml | 6 - .../env/apps/secrets.oauth2-proxy.yaml | 7 - .../fixtures/env/apps/secrets.prometheus.yaml | 9 -- .../sealedsecrets/alerts-secrets.yaml | 18 +++ .../sealedsecrets/cert-manager-secrets.yaml | 43 ++++++ .../sealedsecrets/dns-secrets.yaml | 16 +++ .../sealedsecrets/gitea-secrets.yaml | 16 +++ .../sealedsecrets/grafana-secrets.yaml | 16 +++ .../sealedsecrets/harbor-secrets.yaml | 24 ++++ .../sealedsecrets/keycloak-secrets.yaml | 16 +++ .../kubeflow-pipelines-secrets.yaml | 16 +++ .../sealedsecrets/loki-secrets.yaml | 16 +++ .../oauth2-proxy-redis-secrets.yaml | 16 +++ .../sealedsecrets/oauth2-proxy-secrets.yaml | 16 +++ .../sealedsecrets/obj-secrets.yaml | 16 +++ .../sealedsecrets/oidc-secrets.yaml | 16 +++ .../sealedsecrets/otomi-secrets.yaml | 18 +++ .../sealedsecrets/prometheus-secrets.yaml | 16 +++ .../sealedsecrets/smtp-secrets.yaml | 16 +++ .../team-admin-settings-secrets.yaml | 16 +++ .../team-demo-settings-secrets.yaml | 16 +++ .../team-dev-settings-secrets.yaml | 16 +++ tests/fixtures/env/settings/kms.yaml | 1 + .../fixtures/env/settings/secrets.alerts.yaml | 10 -- tests/fixtures/env/settings/secrets.dns.yaml | 8 -- tests/fixtures/env/settings/secrets.kms.yaml | 8 -- tests/fixtures/env/settings/secrets.obj.yaml | 8 -- tests/fixtures/env/settings/secrets.oidc.yaml | 6 - .../fixtures/env/settings/secrets.otomi.yaml | 10 -- tests/fixtures/env/settings/secrets.smtp.yaml | 6 - .../env/teams/admin/secrets.settings.yaml | 6 - .../env/teams/demo/secrets.settings.yaml | 9 -- .../env/teams/dev/secrets.settings.yaml | 6 - 43 files changed, 471 insertions(+), 302 deletions(-) delete mode 100644 tests/fixtures/env/apps/secrets.cert-manager.yaml delete mode 100644 tests/fixtures/env/apps/secrets.gitea.yaml delete mode 100644 tests/fixtures/env/apps/secrets.grafana.yaml delete mode 100644 tests/fixtures/env/apps/secrets.harbor.yaml delete mode 100644 tests/fixtures/env/apps/secrets.keycloak.yaml delete mode 100644 tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml delete mode 100644 tests/fixtures/env/apps/secrets.loki.yaml delete mode 100644 tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml delete mode 100644 tests/fixtures/env/apps/secrets.oauth2-proxy.yaml delete mode 100644 tests/fixtures/env/apps/secrets.prometheus.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml delete mode 100644 tests/fixtures/env/settings/secrets.alerts.yaml delete mode 100644 tests/fixtures/env/settings/secrets.dns.yaml delete mode 100644 tests/fixtures/env/settings/secrets.kms.yaml delete mode 100644 tests/fixtures/env/settings/secrets.obj.yaml delete mode 100644 tests/fixtures/env/settings/secrets.oidc.yaml delete mode 100644 tests/fixtures/env/settings/secrets.otomi.yaml delete mode 100644 tests/fixtures/env/settings/secrets.smtp.yaml delete mode 100644 tests/fixtures/env/teams/admin/secrets.settings.yaml delete mode 100644 tests/fixtures/env/teams/demo/secrets.settings.yaml delete mode 100644 tests/fixtures/env/teams/dev/secrets.settings.yaml diff --git a/src/common/repo.test.ts b/src/common/repo.test.ts index de6c443315..5c950477b9 100644 --- a/src/common/repo.test.ts +++ b/src/common/repo.test.ts @@ -10,7 +10,6 @@ import { getUniqueIdentifierFromFilePath, hasCorrespondingDecryptedFile, renderManifest, - renderManifestForSecrets, saveResourceGroupToFiles, sortTeamConfigArraysByName, sortUserArraysByName, @@ -47,7 +46,6 @@ describe('getFilePath', () => { const data = {} const jsonPath = ['$', 'apps', 'grafana'] expect(getFilePath(fileMap, jsonPath, data, '')).toEqual('/tmp/values/env/apps/grafana.yaml') - expect(getFilePath(fileMap, jsonPath, data, 'secrets.')).toEqual('/tmp/values/env/apps/secrets.grafana.yaml') }) it('should get path for teamA', () => { const fileMap: FileMap = { @@ -234,11 +232,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'teamConfig', 'demo', 'netpols', '[1]'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/teams/demo/netpols/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/teams/demo/netpols/secrets.a.yaml') }) it('should return file path for platfrom dns', () => { @@ -254,11 +249,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/settings/dns.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/settings/secrets.dns.yaml') }) it('should return file path for user', () => { const fileMap: FileMap = { @@ -273,11 +265,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { id: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/users/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/users/secrets.a.yaml') }) }) @@ -692,13 +681,6 @@ describe('AplCatalog', () => { expect(filePath).toBe('/tmp/values/env/catalogs/default.yaml') }) - it('should return the correct secrets file path for a catalog', () => { - const data = { name: 'default', repositoryUrl: 'https://example.com/charts.git' } - const jsonPath = ['$', 'catalogs', 'default'] - const filePath = getFilePath(catalogFileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/values/env/catalogs/secrets.default.yaml') - }) - it('should use the map key for the file name, not data.name', () => { const data = { name: 'production-charts', repositoryUrl: 'https://example.com/charts.git' } const jsonPath = ['$', 'catalogs', 'prod'] @@ -779,17 +761,6 @@ describe('AplCatalog', () => { }) }) - describe('renderManifestForSecrets', () => { - it('should render a secrets manifest for a catalog', () => { - const data = { secretName: 'git-credentials' } - const manifest = renderManifestForSecrets(catalogFileMap, 'default', data) - - expect(manifest.kind).toBe('AplCatalog') - expect(manifest.metadata.name).toBe('default') - expect(manifest.spec).toEqual(data) - }) - }) - describe('saveResourceGroupToFiles', () => { it('should save a single catalog to a file', async () => { const writeValuesToFile = jest.fn() @@ -856,39 +827,6 @@ describe('AplCatalog', () => { ) }) - it('should save catalog secrets to files with secrets prefix', async () => { - const writeValuesToFile = jest.fn() - const valuesPublic = { - catalogs: { - default: { - name: 'default', - repositoryUrl: 'https://github.com/linode/apl-charts.git', - branch: 'main', - enabled: true, - }, - }, - } - const valuesSecrets = { - catalogs: { - default: { - secretName: 'git-credentials', - }, - }, - } - - await saveResourceGroupToFiles(catalogFileMap, valuesPublic, valuesSecrets, { writeValuesToFile }) - - expect(writeValuesToFile).toHaveBeenCalledTimes(2) - expect(writeValuesToFile).toHaveBeenCalledWith( - '/tmp/values/env/catalogs/secrets.default.yaml', - expect.objectContaining({ - kind: 'AplCatalog', - metadata: { name: 'default' }, - spec: valuesSecrets.catalogs.default, - }), - ) - }) - it('should not write anything when there are no catalogs', async () => { const writeValuesToFile = jest.fn() diff --git a/src/common/repo.ts b/src/common/repo.ts index 6257c89f60..5f7177b514 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,7 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, loadYaml } from './utils' +import { getDirNames, getSchemaSecretsPaths, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -462,28 +462,13 @@ export function renderManifest( return manifest } -export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record) { - let spec = data - if (fileMap.resourceGroup === 'users') { - spec = omit(data, ['id', 'name']) - } - return { - kind: fileMap.kind, - metadata: { - name: resourceName, - }, - spec, - } -} - export async function saveResourceGroupToFiles( fileMap: FileMap, valuesPublic: Record, - valuesSecrets: Record, + _valuesSecrets: Record, deps = { writeValuesToFile }, ): Promise { const jsonPathsValuesPublic = jsonpath.nodes(valuesPublic, fileMap.jsonPathExpression) - const jsonPathsvaluesSecrets = jsonpath.nodes(valuesSecrets, fileMap.jsonPathExpression) await Promise.all( jsonPathsValuesPublic.map(async (node) => { @@ -506,22 +491,8 @@ export async function saveResourceGroupToFiles( }), ) - await Promise.all( - jsonPathsvaluesSecrets.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.') - const resourceName = getResourceName(fileMap, nodePath, nodeValue) - const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue) - await deps.writeValuesToFile(filePath, manifest) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) + // Secrets are now stored as SealedSecret manifests via buildSecretToNamespaceMap() + writeSealedSecretManifests() + // No longer writing secrets.*.yaml files } export function getUniqueIdentifierFromFilePath(filePath: string): string { @@ -588,7 +559,10 @@ export function unsetValuesFileSync(envDir: string): string { return valuesPath } -export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { +export async function loadValues( + envDir: string, + deps = { loadToSpec, loadSealedSecretsToSpec }, +): Promise> { const fileMaps = getFileMaps(envDir).filter((map) => map.loadToSpec === true) const spec = {} @@ -597,11 +571,105 @@ export async function loadValues(envDir: string, deps = { loadToSpec }): Promise await deps.loadToSpec(spec, fileMap) }), ) + await deps.loadSealedSecretsToSpec(spec, envDir) sortTeamConfigArraysByName(spec) sortUserArraysByName(spec) return spec } +/** + * Read sealed secret manifests and merge their encryptedData back into the values spec. + * This restores secret values that helmfile templates need at render time. + * + * Uses the values schema (x-secret paths) to correctly map sealed secret data keys + * back to their original dot-paths, since some property names contain underscores + * (e.g., smtp.auth_password) that should NOT be converted to nested paths. + */ +export async function loadSealedSecretsToSpec( + spec: Record, + envDir: string, + deps = { loadYaml, getSchemaSecretsPaths }, +): Promise { + const sealedSecretsGlob = `${envDir}/env/manifests/namespaces/*/sealedsecrets/*.yaml` + const files = globSync(sealedSecretsGlob, { nodir: true }) + if (files.length === 0) return + + // Get team names from spec to expand teamConfig.* paths + const teams = Object.keys(get(spec, 'teamConfig', {})) + const secretPaths = await deps.getSchemaSecretsPaths(teams) + + // Build a lookup: for each group prefix + dataKey → full schema path + // e.g., "apps.harbor" + "core_secret" → "apps.harbor.core.secret" + const dataKeyToPath = buildDataKeyToPathMap(secretPaths) + + for (const filePath of files) { + const manifest = (await deps.loadYaml(filePath)) as Record | undefined + if (!manifest?.spec?.encryptedData) continue + + const { name: secretName = '' } = (manifest.metadata ?? {}) as { name?: string } + const { encryptedData } = manifest.spec as { encryptedData: Record } + + // Determine target path in spec from the secret name + const targetPath = resolveSecretTargetPath(secretName, spec) + if (!targetPath) continue + + for (const [dataKey, value] of Object.entries(encryptedData)) { + const lookupKey = `${targetPath}/${dataKey}` + const fullPath = dataKeyToPath.get(lookupKey) + if (fullPath) { + set(spec, fullPath, value) + } + } + } +} + +/** + * Build a map from "groupPrefix/dataKey" → "full.schema.path". + * The dataKey is derived from the relative path by replacing dots with underscores, + * matching the convention used in buildSecretToNamespaceMap (sealed-secrets.ts). + */ +function buildDataKeyToPathMap(secretPaths: string[]): Map { + const result = new Map() + for (const secretPath of secretPaths) { + const groupPrefix = findGroupPrefix(secretPath) + if (!groupPrefix) continue + const relativePath = secretPath.slice(groupPrefix.length + 1) + if (!relativePath) continue + const dataKey = relativePath.replace(/\./g, '_') + result.set(`${groupPrefix}/${dataKey}`, secretPath) + } + return result +} + +/** + * Find the group prefix for a secret path — mirrors the logic in sealed-secrets.ts. + */ +function findGroupPrefix(secretPath: string): string | undefined { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) return `teamConfig.${teamMatch[1]}` + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `apps.${appsMatch[1]}` + const [firstSegment] = secretPath.split('.') + if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment + return undefined +} + +function resolveSecretTargetPath(secretName: string, spec: Record): string | undefined { + // team-{name}-settings-secrets → teamConfig.{name} + const teamMatch = secretName.match(/^team-(.+)-settings-secrets$/) + if (teamMatch) return `teamConfig.${teamMatch[1]}` + + // {name}-secrets → apps.{name} or {name} + const nameMatch = secretName.match(/^(.+)-secrets$/) + if (!nameMatch) return undefined + + const [, name] = nameMatch + if (get(spec, `apps.${name}`) !== undefined) return `apps.${name}` + if (get(spec, name) !== undefined) return name + + return undefined +} + export function extractTeamDirectory(filePath: string): string { const match = filePath.match(/\/teams\/([^/]+)/) if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) diff --git a/tests/fixtures/env/apps/cert-manager.yaml b/tests/fixtures/env/apps/cert-manager.yaml index be2c7211a3..d484facf18 100644 --- a/tests/fixtures/env/apps/cert-manager.yaml +++ b/tests/fixtures/env/apps/cert-manager.yaml @@ -3,6 +3,28 @@ metadata: name: cert-manager labels: {} spec: + customRootCA: | + -----BEGIN CERTIFICATE----- + MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N + M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX + xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl + g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw + K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 + xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV + AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB + BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 + tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg + ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y + yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv + ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN + FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI + -----END CERTIFICATE----- externallyManagedTlsSecretName: mysecret issuer: externally-managed-tls-secret _rawValues: {} diff --git a/tests/fixtures/env/apps/secrets.cert-manager.yaml b/tests/fixtures/env/apps/secrets.cert-manager.yaml deleted file mode 100644 index 671916d60e..0000000000 --- a/tests/fixtures/env/apps/secrets.cert-manager.yaml +++ /dev/null @@ -1,55 +0,0 @@ -kind: AplApp -spec: - customRootCA: | - -----BEGIN CERTIFICATE----- - MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N - M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX - xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl - g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw - K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 - xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV - AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB - BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 - tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg - ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y - yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv - ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN - FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI - -----END CERTIFICATE----- - customRootCAKey: | - -----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ - ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd - BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K - GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq - pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH - NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 - 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj - VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr - V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT - UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN - joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B - EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ - AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE - 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT - MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw - UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa - DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl - BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM - wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC - bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ - E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K - 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK - CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H - rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX - EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= - -----END RSA PRIVATE KEY----- -name: cert-manager -metadata: - name: cert-manager diff --git a/tests/fixtures/env/apps/secrets.gitea.yaml b/tests/fixtures/env/apps/secrets.gitea.yaml deleted file mode 100644 index 25e72d933c..0000000000 --- a/tests/fixtures/env/apps/secrets.gitea.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - postgresqlPassword: postgresqlPassword -name: gitea -metadata: - name: gitea diff --git a/tests/fixtures/env/apps/secrets.grafana.yaml b/tests/fixtures/env/apps/secrets.grafana.yaml deleted file mode 100644 index 984e5171de..0000000000 --- a/tests/fixtures/env/apps/secrets.grafana.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: grafana -metadata: - name: grafana diff --git a/tests/fixtures/env/apps/secrets.harbor.yaml b/tests/fixtures/env/apps/secrets.harbor.yaml deleted file mode 100644 index 84dcfe2783..0000000000 --- a/tests/fixtures/env/apps/secrets.harbor.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: AplApp -spec: - adminPassword: harborsomesecretvalue - core: - secret: vQFMm9Qk0pTUF3MK - xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi - jobservice: - secret: CfpanIkcGWz3wGLO - registry: - secret: PjHGEdmPhrmNrEkj - credentials: - htpasswd: admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6 - username: admin - password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 - secretKey: somesecretvalue -name: harbor -metadata: - name: harbor diff --git a/tests/fixtures/env/apps/secrets.keycloak.yaml b/tests/fixtures/env/apps/secrets.keycloak.yaml deleted file mode 100644 index 34f328f775..0000000000 --- a/tests/fixtures/env/apps/secrets.keycloak.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - idp: - clientSecret: somsecretvalue -name: keycloak -metadata: - name: keycloak diff --git a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml b/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml deleted file mode 100644 index 953af4f5c9..0000000000 --- a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - rootPassword: mysqlsomesecretvalue -name: kubeflow-pipelines -metadata: - name: kubeflow-pipelines diff --git a/tests/fixtures/env/apps/secrets.loki.yaml b/tests/fixtures/env/apps/secrets.loki.yaml deleted file mode 100644 index fd1fbcf597..0000000000 --- a/tests/fixtures/env/apps/secrets.loki.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: loki -metadata: - name: loki diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml deleted file mode 100644 index 56f2d4ce60..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - password: gkhugxJsPjhbCybH -name: oauth2-proxy-redis -metadata: - name: oauth2-proxy-redis diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml deleted file mode 100644 index 9361e610fe..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - config: - cookieSecret: gkhugxJsPjhbCybH -name: oauth2-proxy -metadata: - name: oauth2-proxy diff --git a/tests/fixtures/env/apps/secrets.prometheus.yaml b/tests/fixtures/env/apps/secrets.prometheus.yaml deleted file mode 100644 index 74b9528e64..0000000000 --- a/tests/fixtures/env/apps/secrets.prometheus.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplApp -spec: - remoteWrite: - rwConfig: - basicAuth: - password: blalalalalal -name: prometheus -metadata: - name: prometheus diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml new file mode 100644 index 0000000000..ad4e1b66c5 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: alerts-secrets + namespace: apl-secrets +spec: + encryptedData: + slack_url: https://hooks.slack.com/services/id + msteams_highPrio: https://xxxxxxx.com + msteams_lowPrio: https://xxxxxxxx.com + template: + immutable: false + metadata: + name: alerts-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml new file mode 100644 index 0000000000..0dd9c28078 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml @@ -0,0 +1,43 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: cert-manager-secrets + namespace: apl-secrets +spec: + encryptedData: + customRootCAKey: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ + ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd + BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K + GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq + pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH + NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 + 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj + VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr + V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT + UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN + joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B + EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ + AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE + 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT + MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw + UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa + DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl + BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM + wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC + bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ + E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K + 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK + CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H + rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX + EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= + -----END RSA PRIVATE KEY----- + template: + immutable: false + metadata: + name: cert-manager-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml new file mode 100644 index 0000000000..fc1b043bc6 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: dns-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx + template: + immutable: false + metadata: + name: dns-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml new file mode 100644 index 0000000000..f9c85ad94d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: gitea-secrets + namespace: apl-secrets +spec: + encryptedData: + postgresqlPassword: postgresqlPassword + template: + immutable: false + metadata: + name: gitea-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml new file mode 100644 index 0000000000..4138e88e3b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: grafana-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: grafana-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml new file mode 100644 index 0000000000..60b72c8778 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml @@ -0,0 +1,24 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: harbor-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: harborsomesecretvalue + core_secret: vQFMm9Qk0pTUF3MK + core_xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi + jobservice_secret: CfpanIkcGWz3wGLO + registry_secret: PjHGEdmPhrmNrEkj + registry_credentials_htpasswd: "admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6" + registry_credentials_username: admin + registry_credentials_password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 + secretKey: somesecretvalue + template: + immutable: false + metadata: + name: harbor-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml new file mode 100644 index 0000000000..9f1dac4209 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: keycloak-secrets + namespace: apl-secrets +spec: + encryptedData: + idp_clientSecret: somsecretvalue + template: + immutable: false + metadata: + name: keycloak-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml new file mode 100644 index 0000000000..949018981b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: kubeflow-pipelines-secrets + namespace: apl-secrets +spec: + encryptedData: + rootPassword: mysqlsomesecretvalue + template: + immutable: false + metadata: + name: kubeflow-pipelines-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml new file mode 100644 index 0000000000..713eb67bd7 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: loki-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: loki-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml new file mode 100644 index 0000000000..0e0e823755 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-redis-secrets + namespace: apl-secrets +spec: + encryptedData: + password: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-redis-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml new file mode 100644 index 0000000000..b47b028e1b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-secrets + namespace: apl-secrets +spec: + encryptedData: + config_cookieSecret: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml new file mode 100644 index 0000000000..e8e1f6336f --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: obj-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_secretAccessKey: somesecretvalue + template: + immutable: false + metadata: + name: obj-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml new file mode 100644 index 0000000000..b764c3fd6d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oidc-secrets + namespace: apl-secrets +spec: + encryptedData: + clientSecret: somesecretvalue + template: + immutable: false + metadata: + name: oidc-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml new file mode 100644 index 0000000000..2c805cb155 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: otomi-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: bladibla + git_password: gitPasswordForTesting + globalPullSecret_password: blablabla + template: + immutable: false + metadata: + name: otomi-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml new file mode 100644 index 0000000000..d8589306cd --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: prometheus-secrets + namespace: apl-secrets +spec: + encryptedData: + remoteWrite_rwConfig_basicAuth_password: blalalalalal + template: + immutable: false + metadata: + name: prometheus-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml new file mode 100644 index 0000000000..b03b2ab05e --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: smtp-secrets + namespace: apl-secrets +spec: + encryptedData: + auth_password: somesecretvalue + template: + immutable: false + metadata: + name: smtp-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml new file mode 100644 index 0000000000..971824fd93 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-admin-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: YTrnkdUsKPcGATfg + template: + immutable: false + metadata: + name: team-admin-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml new file mode 100644 index 0000000000..214aa453c3 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-demo-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: somesecretvalue + template: + immutable: false + metadata: + name: team-demo-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml new file mode 100644 index 0000000000..fd3676c286 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-dev-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: IkdUsKPcGAdanjas + template: + immutable: false + metadata: + name: team-dev-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/settings/kms.yaml b/tests/fixtures/env/settings/kms.yaml index 70e04a7c17..f3dd212daf 100644 --- a/tests/fixtures/env/settings/kms.yaml +++ b/tests/fixtures/env/settings/kms.yaml @@ -6,5 +6,6 @@ spec: sops: azure: clientId: somesecretvalue + clientSecret: somesecretvalue keys: somesecretvalue tenantId: somesecretvalue diff --git a/tests/fixtures/env/settings/secrets.alerts.yaml b/tests/fixtures/env/settings/secrets.alerts.yaml deleted file mode 100644 index 1bbbfdaf12..0000000000 --- a/tests/fixtures/env/settings/secrets.alerts.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplAlertSet -spec: - slack: - url: https://hooks.slack.com/services/id - msteams: - highPrio: https://xxxxxxx.com - lowPrio: https://xxxxxxxx.com -name: alerts -metadata: - name: alerts diff --git a/tests/fixtures/env/settings/secrets.dns.yaml b/tests/fixtures/env/settings/secrets.dns.yaml deleted file mode 100644 index a6024d2e07..0000000000 --- a/tests/fixtures/env/settings/secrets.dns.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplDns -spec: - provider: - linode: - apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx -name: dns -metadata: - name: dns diff --git a/tests/fixtures/env/settings/secrets.kms.yaml b/tests/fixtures/env/settings/secrets.kms.yaml deleted file mode 100644 index 74b33f1f1c..0000000000 --- a/tests/fixtures/env/settings/secrets.kms.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplKms -spec: - sops: - azure: - clientSecret: somesecretvalue -name: kms -metadata: - name: kms diff --git a/tests/fixtures/env/settings/secrets.obj.yaml b/tests/fixtures/env/settings/secrets.obj.yaml deleted file mode 100644 index 82c77762ff..0000000000 --- a/tests/fixtures/env/settings/secrets.obj.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplObjectStorage -spec: - provider: - linode: - secretAccessKey: somesecretvalue -name: obj -metadata: - name: obj diff --git a/tests/fixtures/env/settings/secrets.oidc.yaml b/tests/fixtures/env/settings/secrets.oidc.yaml deleted file mode 100644 index 5f03c2f11d..0000000000 --- a/tests/fixtures/env/settings/secrets.oidc.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplIdentityProvider -spec: - clientSecret: somesecretvalue -name: oidc -metadata: - name: oidc diff --git a/tests/fixtures/env/settings/secrets.otomi.yaml b/tests/fixtures/env/settings/secrets.otomi.yaml deleted file mode 100644 index 1819165b7c..0000000000 --- a/tests/fixtures/env/settings/secrets.otomi.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplCapabilitySet -spec: - adminPassword: bladibla - git: - password: gitPasswordForTesting - globalPullSecret: - password: blablabla -name: otomi -metadata: - name: otomi diff --git a/tests/fixtures/env/settings/secrets.smtp.yaml b/tests/fixtures/env/settings/secrets.smtp.yaml deleted file mode 100644 index 62a1d33e38..0000000000 --- a/tests/fixtures/env/settings/secrets.smtp.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplSmtp -spec: - auth_password: somesecretvalue -name: smtp -metadata: - name: smtp diff --git a/tests/fixtures/env/teams/admin/secrets.settings.yaml b/tests/fixtures/env/teams/admin/secrets.settings.yaml deleted file mode 100644 index 595f37ad22..0000000000 --- a/tests/fixtures/env/teams/admin/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: YTrnkdUsKPcGATfg -name: admin -metadata: - name: admin diff --git a/tests/fixtures/env/teams/demo/secrets.settings.yaml b/tests/fixtures/env/teams/demo/secrets.settings.yaml deleted file mode 100644 index abf4440deb..0000000000 --- a/tests/fixtures/env/teams/demo/secrets.settings.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: somesecretvalue - alerts: - slack: - url: https://slack.con -name: demo -metadata: - name: demo diff --git a/tests/fixtures/env/teams/dev/secrets.settings.yaml b/tests/fixtures/env/teams/dev/secrets.settings.yaml deleted file mode 100644 index 41599125ef..0000000000 --- a/tests/fixtures/env/teams/dev/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: IkdUsKPcGAdanjas -name: dev -metadata: - name: dev From c0fea6e795cb736f2e7108c21253985914ec112b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:29:05 +0100 Subject: [PATCH 56/71] fix: improve password retrieval logic in getRepo function --- src/common/git-config.ts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index bf15462522..a19130f267 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -152,21 +152,29 @@ export const getRepo = async (values: Record, deps = { getK8sSecret otomiGit.repoUrl = process.env.GIT_REPO_URL } const username = otomiGit?.username - let password = otomiGit?.password ?? '' + let password = '' const email = otomiGit?.email const branch = otomiGit?.branch - // If password is missing or is an unresolved sealed-secret placeholder, - // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) - if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { - try { - const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) - if (secret?.git_password) { - password = String(secret.git_password) - d.debug('Read git password from K8s secret (ESO)') - } - } catch { - d.warn('Could not read git password from K8s secret, using value from config') + // Always try the K8s secret first for the real password. + // Values may contain encrypted sealed-secret ciphertext (from loadSealedSecretsToSpec) + // which must not be used as the actual git password. + try { + const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) + if (secret?.git_password) { + password = String(secret.git_password) + d.debug('Read git password from K8s secret (ESO)') + } + } catch { + d.debug('Could not read git password from K8s secret') + } + + // Fall back to values if K8s secret is not available (e.g., during bootstrap) + if (!password) { + const valuesPassword = otomiGit?.password ?? '' + // Only use the values password if it's not an unresolved sealed-secret placeholder + if (valuesPassword && !(typeof valuesPassword === 'string' && valuesPassword.startsWith('sealed:'))) { + password = valuesPassword } } From 4c8ed8234e84104a03adcceb881650eb8b01047a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:46:06 +0100 Subject: [PATCH 57/71] fix: add new namespaces in core.yaml --- core.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core.yaml b/core.yaml index 9023df3cf7..bf5568ad84 100644 --- a/core.yaml +++ b/core.yaml @@ -87,6 +87,11 @@ k8s: disablePolicyChecks: true - name: sealed-secrets app: sealed-secrets + - name: external-secrets + app: external-secrets + disableIstioInjection: true + - name: apl-secrets + disableIstioInjection: true - name: policy-reporter app: policy-reporter disablePolicyChecks: true From 19405755eb00ba4c1a89a55c5735eb74f7e2a0c3 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:45:48 +0200 Subject: [PATCH 58/71] fix: conditional rewrite rules for ingress --- values/k8s/k8s-raw.gotmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 1eec03a63e..7ebb4eef16 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -83,5 +83,9 @@ resources: data: otomi-hairpin.include: | {{- $escapedDomain := $v.cluster.domainSuffix | replace "." "\\." }} + {{- if $v.apps | get "ingress-nginx-platform.enabled" false }} rewrite name regex (.+)\.{{ $escapedDomain }} ingress-nginx-platform-controller.ingress.svc.cluster.local answer auto + {{- else }} + rewrite name regex (.+)\.{{ $escapedDomain }} {{ $v.ingress.platformClass.className }}-istio.istio-system.svc.cluster.local answer auto + {{- end }} {{- end }} From 8d0a018cd1737842119d509d88cd65b584da9312 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:36:15 +0200 Subject: [PATCH 59/71] fix: harbor registry username --- values/harbor/harbor-raw.gotmpl | 5 +++++ values/harbor/harbor.gotmpl | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index 6d36bb30ba..a6d199dc1a 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -72,6 +72,7 @@ resources: data: REGISTRY_PASSWD: '{{ "{{ .password | toString }}" }}' REGISTRY_HTPASSWD: '{{ "{{ .htpasswd | toString }}" }}' + REGISTRY_USERNAME: '{{ "{{ .username | toString }}" }}' data: - secretKey: password remoteRef: @@ -81,6 +82,10 @@ resources: remoteRef: key: harbor-secrets property: registry_credentials_htpasswd + - secretKey: username + remoteRef: + key: harbor-secrets + property: registry_credentials_username - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: diff --git a/values/harbor/harbor.gotmpl b/values/harbor/harbor.gotmpl index 5e1f10a742..c871d9e649 100644 --- a/values/harbor/harbor.gotmpl +++ b/values/harbor/harbor.gotmpl @@ -28,6 +28,12 @@ core: existingSecret: harbor-core-secret existingXsrfSecret: harbor-core-xsrf-secret existingXsrfSecretKey: CSRF_KEY + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME database: maxOpenConns: {{ $h.databaseMaxConnections }} @@ -73,6 +79,12 @@ jobservice: - stdout resources: {{- $h.resources.jobservice | toYaml | nindent 4 }} existingSecret: harbor-jobservice-secret + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME metrics: serviceMonitor: From ac53f2f66b8c6acea9122d6a96d1338fb969c48f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 9 Apr 2026 11:24:44 +0200 Subject: [PATCH 60/71] chore: add sealed secrets timeout and interval configuration --- src/cmd/install.ts | 4 ++-- src/cmd/migrate.ts | 5 +++-- src/common/envalid.ts | 8 ++++++++ src/common/sealed-secrets.ts | 7 +++---- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 01b2b035c5..0eb58c6bbd 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -79,8 +79,8 @@ const allSecretsExist = async (secrets: AppliedSecret[], deps = { getK8sSecret } const waitForSealedSecrets = async ( appliedSecrets: AppliedSecret[], - timeoutMs = 120000, - intervalMs = 3000, + timeoutMs = env.SEALED_SECRETS_TIMEOUT_MS, + intervalMs = env.SEALED_SECRETS_INTERVAL_MS, deps = { getK8sSecret, terminal }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 0ac0538b39..9dbbd9fb8b 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -37,6 +37,7 @@ import { } from '../common/sealed-secrets' const cmdName = getFilename(__filename) +const sealedSecretManifestsGlob = `${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml` interface Arguments extends BasicArguments { dryRun?: boolean @@ -1047,7 +1048,7 @@ export const sopsMigration = async ( // them, or the controller used its auto-generated key), re-apply them and restart // the controller so subsequent steps can resolve the git password. if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + const existingManifests = deps.globSync(sealedSecretManifestsGlob, { dot: false, }) if (existingManifests.length > 0) { @@ -1067,7 +1068,7 @@ export const sopsMigration = async ( } // Secondary guard: if manifests already exist, just clean up SOPS artifacts - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + const existingManifests = deps.globSync(sealedSecretManifestsGlob, { dot: false, }) if (existingManifests.length > 0) { diff --git a/src/common/envalid.ts b/src/common/envalid.ts index 64d5a81858..3f2f7d6d1e 100644 --- a/src/common/envalid.ts +++ b/src/common/envalid.ts @@ -66,6 +66,14 @@ export const cliEnvSpec = { desc: 'The URL for the default APL catalog', default: 'https://github.com/linode/apl-charts.git', }), + SEALED_SECRETS_TIMEOUT_MS: num({ + desc: 'Timeout in milliseconds for waiting on sealed secrets operations', + default: 120000, + }), + SEALED_SECRETS_INTERVAL_MS: num({ + desc: 'Polling interval in milliseconds for sealed secrets readiness checks', + default: 3000, + }), } export function cleanEnv(spec: { [K in keyof T]: ValidatorSpec }, options?: CleanOptions) { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 1f92b9e4e1..be14ca84d7 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -8,14 +8,13 @@ import { pki } from 'node-forge' import { join } from 'path' import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' import { terminal } from 'src/common/debug' +import { env } from 'src/common/envalid' import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' -const ROLLOUT_TIMEOUT_MS = 120000 -const ROLLOUT_INTERVAL_MS = 3000 /** * Strip ALL x-secret fields from values before writing to disk. @@ -498,7 +497,7 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi d.info('Waiting for sealed-secrets controller rollout') const start = Date.now() - while (Date.now() - start < ROLLOUT_TIMEOUT_MS) { + while (Date.now() - start < env.SEALED_SECRETS_TIMEOUT_MS) { try { const deployment = await k8s.app().readNamespacedDeployment({ name: 'sealed-secrets', @@ -514,7 +513,7 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi } catch { // Ignore transient read errors during rollout } - await new Promise((resolve) => setTimeout(resolve, ROLLOUT_INTERVAL_MS)) + await new Promise((resolve) => setTimeout(resolve, env.SEALED_SECRETS_INTERVAL_MS)) } d.warn('Rollout status check timed out') } From 783905260654e80612234ff784fe21478ee3649e Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 9 Apr 2026 11:27:06 +0200 Subject: [PATCH 61/71] chore: refactor secret management for git configuration and add team secrets template --- charts/otomi-api/templates/deployment.yaml | 4 +- charts/otomi-api/values.yaml | 3 +- helmfile.d/helmfile-60.teams.yaml.gotmpl | 121 +------------------ helmfile.d/snippets/grafana.gotmpl | 2 + values/otomi-api/otomi-api.gotmpl | 3 +- values/team-secrets/team-secrets-raw.gotmpl | 125 ++++++++++++++++++++ 6 files changed, 134 insertions(+), 124 deletions(-) create mode 100644 values/team-secrets/team-secrets-raw.gotmpl diff --git a/charts/otomi-api/templates/deployment.yaml b/charts/otomi-api/templates/deployment.yaml index 0e75379733..aaf05d74f7 100644 --- a/charts/otomi-api/templates/deployment.yaml +++ b/charts/otomi-api/templates/deployment.yaml @@ -40,7 +40,7 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} - {{- with .Values.existingSecret }} + {{- with .Values.gitConfig.existingSecret }} - secretRef: name: {{ . }} {{- end }} @@ -81,7 +81,7 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} - {{- with .Values.existingSecret }} + {{- with .Values.gitConfig.existingSecret }} - secretRef: name: {{ . }} {{- end }} diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index 3e40a81229..e92b834e24 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -24,7 +24,8 @@ serviceAccount: imagePullSecrets: {} -existingSecret: "" +gitConfig: + existingSecret: "" rbac: # Specifies whether rbac should be set up diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index 3ab74b4ed6..767d1a3ce6 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -15,8 +15,6 @@ bases: {{- $tc := $v.teamConfig }} {{- $domain := ($v.cluster | get "domainSuffix" nil) }} {{- $gatewayName := $v.ingress.platformClass.className }} -{{- $slackTpl := tpl (readFile "../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} -{{- $opsgenieTpl := tpl (readFile "../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} {{- $httpRoute := tpl (readFile "../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} releases: {{- range $teamId, $team := omit $tc "admin" }} @@ -26,8 +24,6 @@ releases: {{- $grafanaHostname := printf "grafana-%s.%s" $teamId $domain }} {{- $tektonHostname := printf "tekton-%s.%s" $teamId $domain }} {{- $teamApps := index $tc $teamId "apps" | default dict }} - {{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} - {{- $teamAlertmanagerConfig := tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} - name: tekton-dashboard-{{ $teamId }} installed: {{ $a | get "tekton.enabled" }} namespace: team-{{ $teamId }} @@ -234,120 +230,5 @@ releases: team: {{ $teamId }} pipeline: otomi-task-teams values: - - resources: - {{- if $teamSettings | get "managedMonitoring.grafana" false }} - - apiVersion: external-secrets.io/v1beta1 - kind: ExternalSecret - metadata: - name: team-{{ $teamId }}-grafana-admin - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: team-{{ $teamId }}-grafana-admin - creationPolicy: Owner - template: - type: Opaque - data: - admin-user: {{ $teamId }} - admin-password: '{{ "{{ .password | toString }}" }}' - data: - - secretKey: password - remoteRef: - key: team-{{ $teamId }}-settings-secrets - property: settings_password - {{- end }} - {{- if $teamSettings | get "managedMonitoring.grafana" false }} - - apiVersion: external-secrets.io/v1beta1 - kind: ExternalSecret - metadata: - name: grafana-oidc-secret - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: grafana-oidc-secret - creationPolicy: Owner - template: - type: Opaque - data: - client_id: {{ $v.apps.keycloak.idp.clientID }} - client_secret: '{{ "{{ .clientSecret | toString }}" }}' - data: - - secretKey: clientSecret - remoteRef: - key: keycloak-secrets - property: idp_clientSecret - - apiVersion: external-secrets.io/v1beta1 - kind: ExternalSecret - metadata: - name: grafana-loki-datasource-secret - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: grafana-loki-datasource-secret - creationPolicy: Owner - template: - type: Opaque - data: - password: '{{ "{{ .adminPassword | toString }}" }}' - data: - - secretKey: adminPassword - remoteRef: - key: loki-secrets - property: adminPassword - {{- end }} - {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} - - apiVersion: external-secrets.io/v1beta1 - kind: ExternalSecret - metadata: - name: alertmanager-team-{{ $teamId }}-config - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: alertmanager-team-{{ $teamId }}-config - creationPolicy: Owner - template: - type: Opaque - data: - alertmanager.yaml: | - {{- $teamAlertmanagerConfig | nindent 20 }} - data: - {{- if has "slack" $teamReceivers }} - - secretKey: slackUrl - remoteRef: - key: alerts-secrets - property: slack_url - {{- end }} - {{- if has "email" $teamReceivers }} - - secretKey: smtpAuthPassword - remoteRef: - key: smtp-secrets - property: auth_password - - secretKey: smtpAuthSecret - remoteRef: - key: smtp-secrets - property: auth_secret - {{- end }} - {{- if has "opsgenie" $teamReceivers }} - - secretKey: opsgenieApiKey - remoteRef: - key: alerts-secrets - property: opsgenie_apiKey - {{- end }} - {{- end }} + - ../values/team-secrets/team-secrets-raw.gotmpl {{- end }} diff --git a/helmfile.d/snippets/grafana.gotmpl b/helmfile.d/snippets/grafana.gotmpl index 4490a0f2fd..95c64bebd7 100644 --- a/helmfile.d/snippets/grafana.gotmpl +++ b/helmfile.d/snippets/grafana.gotmpl @@ -9,6 +9,8 @@ analytics: org_role: Admin allow_sign_up: true oauth_auto_login: true + # $__env{} is Grafana's built-in environment variable interpolation for config values. + # See: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion client_id: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_ID} client_secret: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} scopes: email profile openid diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index 37cb1894ba..ae8fe79618 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -37,7 +37,8 @@ tools: secrets: GIT_USER: {{ $git.username | quote }} GIT_EMAIL: {{ $git.email | quote }} -existingSecret: otomi-api-git-credentials +gitConfig: + existingSecret: otomi-api-git-credentials env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }} diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl new file mode 100644 index 0000000000..1b2f66c1b8 --- /dev/null +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -0,0 +1,125 @@ +{{- $v := .Values }} +{{- $teamId := .Release.Labels.team }} +{{- $tc := $v.teamConfig }} +{{- $teamSettings := (index $tc $teamId).settings }} +{{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} +{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} +{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} +{{- $teamAlertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} + +resources: +{{- if $teamSettings | get "managedMonitoring.grafana" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: team-{{ $teamId }}-grafana-admin + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: team-{{ $teamId }}-grafana-admin + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: {{ $teamId }} + admin-password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: team-{{ $teamId }}-settings-secrets + property: settings_password +{{- end }} +{{- if $teamSettings | get "managedMonitoring.grafana" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $v.apps.keycloak.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword +{{- end }} +{{- if $teamSettings | get "managedMonitoring.alertmanager" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-team-{{ $teamId }}-config + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-team-{{ $teamId }}-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $teamAlertmanagerConfig | nindent 12 }} + data: + {{- if has "slack" $teamReceivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $teamReceivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} +{{- end }} From 42654ab67e027e31cef6b7f37bce83764e558438 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:10:02 +0200 Subject: [PATCH 62/71] fix: prevent overwriting specVersion in default values during migration --- src/operator/apl-operator.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index b3c87cec13..5e395bd112 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -76,6 +76,11 @@ export class AplOperator { try { const defaultValues = (await hfValues({ defaultValues: true })) as Record + // specVersion is managed by the migration system — do not overwrite it with defaults. + // migrate() compares the on-disk specVersion against values-changes.yaml to detect pending migrations. + if (defaultValues?.versions) { + delete defaultValues.versions.specVersion + } this.d.info('Write default values to env repo') await writeValues(defaultValues) From 5117fbd4f8361c4cf9cf7fce8b9fa8d991a1957f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:52:41 +0200 Subject: [PATCH 63/71] revert: remove specVersion overwriting in default values during migration --- src/operator/apl-operator.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 5e395bd112..b3c87cec13 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -76,11 +76,6 @@ export class AplOperator { try { const defaultValues = (await hfValues({ defaultValues: true })) as Record - // specVersion is managed by the migration system — do not overwrite it with defaults. - // migrate() compares the on-disk specVersion against values-changes.yaml to detect pending migrations. - if (defaultValues?.versions) { - delete defaultValues.versions.specVersion - } this.d.info('Write default values to env repo') await writeValues(defaultValues) From b8ec06d184e1f0e0fc8f5ad49b765d056256c0c6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 9 Apr 2026 17:49:22 +0200 Subject: [PATCH 64/71] fix: sops to sealed secrets migration --- src/cmd/migrate.test.ts | 10 ---------- src/cmd/migrate.ts | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index db8d8ffcbe..1f84fff8e0 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -992,16 +992,6 @@ describe('sopsMigration', () => { expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() }) - it('should only clean up when manifests already exist', async () => { - mockExistsSync.mockReturnValue(true) - mockGlobSync.mockReturnValue(['/some/manifest.yaml']) - - await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) - - expect(mockRemoveSopsArtifacts).toHaveBeenCalled() - expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() - }) - it('should run full migration path', async () => { mockExistsSync.mockReturnValue(true) mockGlobSync.mockReturnValue([]) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 9dbbd9fb8b..261e3f20e6 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -1067,16 +1067,6 @@ export const sopsMigration = async ( return } - // Secondary guard: if manifests already exist, just clean up SOPS artifacts - const existingManifests = deps.globSync(sealedSecretManifestsGlob, { - dot: false, - }) - if (existingManifests.length > 0) { - d.info('SealedSecret manifests already exist, only cleaning up SOPS artifacts') - deps.removeSopsArtifacts() - return - } - d.info('Starting SOPS to SealedSecrets migration') // Get or generate sealed-secrets key From d7f517bd2a99d174687af61a075d12c991c0c3f2 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 13 Apr 2026 09:24:59 +0200 Subject: [PATCH 65/71] fix: address PR comments --- helmfile.d/snippets/derived.gotmpl | 1 + src/cmd/bootstrap.ts | 9 +++--- src/common/sealed-secrets.test.ts | 5 ++- src/common/sealed-secrets.ts | 8 +---- .../gitea-db-secret-raw.gotmpl | 2 +- values/ingress-nginx/ingress-nginx-raw.gotmpl | 32 ------------------- values/k8s/k8s-raw.gotmpl | 17 ---------- 7 files changed, 10 insertions(+), 64 deletions(-) diff --git a/helmfile.d/snippets/derived.gotmpl b/helmfile.d/snippets/derived.gotmpl index 87739ce71b..9f5bf220bd 100644 --- a/helmfile.d/snippets/derived.gotmpl +++ b/helmfile.d/snippets/derived.gotmpl @@ -200,6 +200,7 @@ environments: ttyDomain: {{ print "tty." $domainSuffix }} consoleDomain: {{ printf "console.%s" $domainSuffix }} giteaDomain: {{ printf "gitea.%s" $domainSuffix }} + giteaDbUsername: gitea keycloakDomain: {{ printf "keycloak.%s" $domainSuffix }} harborDomain: {{ printf "harbor.%s" $domainSuffix }} tlsSecretName: {{ $tlsSecretName }} diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 2a4f0cb527..d28a1fc749 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -186,18 +186,19 @@ export const processValues = async ( // Store users in allSecrets for sealed secret generation // The keycloak-operator derives groups from isPlatformAdmin/isTeamAdmin/teams directly allSecrets.users = users + // Include users in originalInput — getUsers() may return a detached array + // when originalInput had no 'users' key initially + const newInput = merge(cloneDeep(originalInput), cloneDeep({ users })) // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) - const mergedForDisk = merge(cloneDeep(originalInput), cloneDeep(allSecrets), cloneDeep({ users })) + const mergedForDisk = merge(cloneDeep(newInput), cloneDeep(allSecrets)) const secretPaths = await deps.getSchemaSecretsPaths(Object.keys(get(mergedForDisk, 'teamConfig', {}))) const valuesForDisk = deps.stripAllSecrets(mergedForDisk, secretPaths) await deps.writeValues(valuesForDisk) // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - // Include users (with name/UUID) on originalInput for bootstrapSealedSecrets to find them. - // getUsers() may return a detached array when originalInput had no 'users' key initially. - return { originalInput: { ...originalInput, users }, allSecrets } + return { originalInput: newInput, allSecrets } } // create file structure based on file entry diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index cee5b3ab0e..69dd2c6e96 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -586,14 +586,13 @@ describe('sealed-secrets', () => { expect(mockK8s.custom().patchNamespacedCustomObject).toHaveBeenCalled() }) - it('should log error when manifests fail to apply', async () => { + it('should throw when manifests fail to apply', async () => { const { k8s: mockK8s } = require('src/common/k8s') mockK8s.custom().patchNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) const manifests = [makeMockManifest('test-secret', 'apl-secrets')] - await applySealedSecretManifests(manifests, { terminal }) - // Should not throw, just log the error + await expect(applySealedSecretManifests(manifests, { terminal })).rejects.toThrow('apply failed') }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index be14ca84d7..9b514709c1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -257,8 +257,6 @@ export const buildSecretToNamespaceMap = async ( const groupMap = new Map() for (const secretPath of secretPaths) { - // Skip SOPS-related paths - if (secretPath.startsWith('kms.sops')) continue // Skip users path — user secrets are managed individually in apl-users namespace if (secretPath === 'users') continue @@ -396,11 +394,7 @@ export const applySealedSecretManifests = async ( for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) - try { - await applySealedSecretResource(manifest) - } catch (error) { - d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) - } + await applySealedSecretResource(manifest) } } diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 067dfb8644..7f2b9e4f2a 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -16,7 +16,7 @@ resources: template: type: kubernetes.io/basic-auth data: - username: gitea + username: {{ $v._derived.giteaDbUsername }} password: '{{ "{{ .postgresqlPassword | toString }}" }}' data: - secretKey: postgresqlPassword diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 366b4d6389..7ee4eef902 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -13,36 +13,4 @@ resources: {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" -{{- end }} -# ClusterRole to allow ingress controller to read TLS secrets from all namespaces -{{- range $ingress := $v.ingress.classes }} -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} - rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - subjects: - - kind: ServiceAccount - name: ingress-nginx-{{ $ingress.className }} - namespace: ingress {{- end }} \ No newline at end of file diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 7ebb4eef16..65631e6f26 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -72,20 +72,3 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." - {{- if $v.cluster.domainSuffix }} - # CoreDNS custom config to resolve platform domains to the ingress controller ClusterIP. - # This avoids hairpin NAT issues where pods cannot reach services via the external LoadBalancer IP. - - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns-custom - namespace: kube-system - data: - otomi-hairpin.include: | - {{- $escapedDomain := $v.cluster.domainSuffix | replace "." "\\." }} - {{- if $v.apps | get "ingress-nginx-platform.enabled" false }} - rewrite name regex (.+)\.{{ $escapedDomain }} ingress-nginx-platform-controller.ingress.svc.cluster.local answer auto - {{- else }} - rewrite name regex (.+)\.{{ $escapedDomain }} {{ $v.ingress.platformClass.className }}-istio.istio-system.svc.cluster.local answer auto - {{- end }} - {{- end }} From bd56ed9ece9ac05719d474ebf8b38e56ce2f3c98 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 14 Apr 2026 10:16:09 +0200 Subject: [PATCH 66/71] fix: address PR comments --- src/cmd/migrate.test.ts | 7 +++++ src/cmd/migrate.ts | 20 ++++++-------- src/common/sealed-secrets.ts | 53 ++++++++++++++++++++++++------------ 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 1f84fff8e0..b59ac980cf 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -937,6 +937,13 @@ describe('sopsMigration', () => { existsSync: mockExistsSync, globSync: mockGlobSync, terminal: mockTerminal, + getOrCreateSealedSecretsPem: jest.fn().mockImplementation(async (innerDeps: any) => { + const cert = await innerDeps.getExistingSealedSecretsCert() + if (cert) return innerDeps.getPemFromCertificate(cert) + const { certificate, privateKey } = innerDeps.generateSealedSecretsKeyPair() + await innerDeps.createSealedSecretsKeySecret(certificate, privateKey) + return innerDeps.getPemFromCertificate(certificate) + }), getExistingSealedSecretsCert: mockGetExistingSealedSecretsCert, getPemFromCertificate: mockGetPemFromCertificate, generateSealedSecretsKeyPair: mockGenerateSealedSecretsKeyPair, diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 4a3d502dda..d507e21b67 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -30,6 +30,7 @@ import { createUserSealedSecretManifests, generateSealedSecretsKeyPair, getExistingSealedSecretsCert, + getOrCreateSealedSecretsPem, getPemFromCertificate, restartSealedSecretsController, SealedSecretManifest, @@ -1015,6 +1016,7 @@ export const sopsMigration = async ( existsSync, globSync, terminal, + getOrCreateSealedSecretsPem, getExistingSealedSecretsCert, getPemFromCertificate, generateSealedSecretsKeyPair, @@ -1060,17 +1062,13 @@ export const sopsMigration = async ( d.info('Starting SOPS to SealedSecrets migration') // Get or generate sealed-secrets key - let pem: string - const existingCert = await deps.getExistingSealedSecretsCert() - if (existingCert) { - d.info('Using existing sealed-secrets certificate') - pem = deps.getPemFromCertificate(existingCert) - } else { - d.info('Generating new sealed-secrets key pair') - const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() - await deps.createSealedSecretsKeySecret(certificate, privateKey) - pem = deps.getPemFromCertificate(certificate) - } + const pem = await deps.getOrCreateSealedSecretsPem({ + terminal: deps.terminal, + getExistingSealedSecretsCert: deps.getExistingSealedSecretsCert, + getPemFromCertificate: deps.getPemFromCertificate, + generateSealedSecretsKeyPair: deps.generateSealedSecretsKeyPair, + createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, + }) // Build secret-to-namespace mappings const teams = Object.keys((values.teamConfig as Record) || {}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 9b514709c1..d2618624f2 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -15,6 +15,7 @@ import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' +const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' /** * Strip ALL x-secret fields from values before writing to disk. @@ -340,7 +341,7 @@ export const writeSealedSecretManifests = async ( const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) for (const manifest of manifests) { - const dir = `${envDir}/env/manifests/namespaces/${manifest.metadata.namespace}/sealedsecrets` + const dir = `${envDir}/${SEALED_SECRETS_MANIFESTS_SUBDIR}/${manifest.metadata.namespace}/sealedsecrets` await deps.mkdir(dir, { recursive: true }) const filePath = `${dir}/${manifest.metadata.name}.yaml` d.info(`Writing sealed secret to ${filePath}`) @@ -411,7 +412,7 @@ export const applySealedSecretManifestsFromDir = async ( deps = { terminal, readdir, readFile, existsSync }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) - const manifestsDir = join(envDir, 'env/manifests/namespaces') + const manifestsDir = join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR) if (!deps.existsSync(manifestsDir)) { d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) @@ -570,6 +571,31 @@ export const createUserSealedSecretManifests = async ( return manifests } +/** + * Get the PEM public key from the existing sealed-secrets certificate in the cluster, + * or generate a new RSA key pair, store it in the cluster, and return its PEM. + */ +export const getOrCreateSealedSecretsPem = async ( + deps = { + terminal, + getExistingSealedSecretsCert, + getPemFromCertificate, + generateSealedSecretsKeyPair, + createSealedSecretsKeySecret, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:getOrCreateSealedSecretsPem`) + const existingCert = await deps.getExistingSealedSecretsCert() + if (existingCert) { + d.info('Using existing sealed-secrets certificate') + return deps.getPemFromCertificate(existingCert) + } + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + return deps.getPemFromCertificate(certificate) +} + export const bootstrapSealedSecrets = async ( secrets: Record, envDir: string, @@ -590,21 +616,14 @@ export const bootstrapSealedSecrets = async ( const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) d.info('Bootstrapping sealed secrets') - // 1. Check if there's an existing sealed-secrets key in the cluster - const existingCert = await deps.getExistingSealedSecretsCert() - - let pem: string - if (existingCert) { - // Use existing certificate for encryption - d.info('Using existing sealed-secrets certificate') - pem = deps.getPemFromCertificate(existingCert) - } else { - // Generate new key pair and create the secret - d.info('Generating new sealed-secrets key pair') - const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() - await deps.createSealedSecretsKeySecret(certificate, privateKey) - pem = deps.getPemFromCertificate(certificate) - } + // 1. Get or create the sealed-secrets PEM public key + const pem = await getOrCreateSealedSecretsPem({ + terminal: deps.terminal, + getExistingSealedSecretsCert: deps.getExistingSealedSecretsCert, + getPemFromCertificate: deps.getPemFromCertificate, + generateSealedSecretsKeyPair: deps.generateSealedSecretsKeyPair, + createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, + }) // 5. Build secret-to-namespace mapping const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) From 632b8d68a20e1bf0f41cb6941fa3b58c2daabcfc Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:56:50 +0200 Subject: [PATCH 67/71] fix: remove loadSealedSecretsToSpec --- src/common/git-config.ts | 3 +- src/common/repo.ts | 101 +----------------- .../fixtures/env/apps/kubeflow-pipelines.yaml | 1 + tests/fixtures/env/settings/alerts.yaml | 4 +- tests/fixtures/env/settings/obj.yaml | 1 + tests/fixtures/env/settings/otomi.yaml | 1 + 6 files changed, 9 insertions(+), 102 deletions(-) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 2bc8c5e69f..1fefc4fc48 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -164,8 +164,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret const branch = otomiGit?.branch // Always try the K8s secret first for the real password. - // Values may contain encrypted sealed-secret ciphertext (from loadSealedSecretsToSpec) - // which must not be used as the actual git password. + // On disk, secrets are stripped — the values spec has no plaintext password. try { const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) if (secret?.git_password) { diff --git a/src/common/repo.ts b/src/common/repo.ts index 5f7177b514..ab89e6060b 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,7 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, getSchemaSecretsPaths, loadYaml } from './utils' +import { getDirNames, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -559,10 +559,7 @@ export function unsetValuesFileSync(envDir: string): string { return valuesPath } -export async function loadValues( - envDir: string, - deps = { loadToSpec, loadSealedSecretsToSpec }, -): Promise> { +export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { const fileMaps = getFileMaps(envDir).filter((map) => map.loadToSpec === true) const spec = {} @@ -571,105 +568,11 @@ export async function loadValues( await deps.loadToSpec(spec, fileMap) }), ) - await deps.loadSealedSecretsToSpec(spec, envDir) sortTeamConfigArraysByName(spec) sortUserArraysByName(spec) return spec } -/** - * Read sealed secret manifests and merge their encryptedData back into the values spec. - * This restores secret values that helmfile templates need at render time. - * - * Uses the values schema (x-secret paths) to correctly map sealed secret data keys - * back to their original dot-paths, since some property names contain underscores - * (e.g., smtp.auth_password) that should NOT be converted to nested paths. - */ -export async function loadSealedSecretsToSpec( - spec: Record, - envDir: string, - deps = { loadYaml, getSchemaSecretsPaths }, -): Promise { - const sealedSecretsGlob = `${envDir}/env/manifests/namespaces/*/sealedsecrets/*.yaml` - const files = globSync(sealedSecretsGlob, { nodir: true }) - if (files.length === 0) return - - // Get team names from spec to expand teamConfig.* paths - const teams = Object.keys(get(spec, 'teamConfig', {})) - const secretPaths = await deps.getSchemaSecretsPaths(teams) - - // Build a lookup: for each group prefix + dataKey → full schema path - // e.g., "apps.harbor" + "core_secret" → "apps.harbor.core.secret" - const dataKeyToPath = buildDataKeyToPathMap(secretPaths) - - for (const filePath of files) { - const manifest = (await deps.loadYaml(filePath)) as Record | undefined - if (!manifest?.spec?.encryptedData) continue - - const { name: secretName = '' } = (manifest.metadata ?? {}) as { name?: string } - const { encryptedData } = manifest.spec as { encryptedData: Record } - - // Determine target path in spec from the secret name - const targetPath = resolveSecretTargetPath(secretName, spec) - if (!targetPath) continue - - for (const [dataKey, value] of Object.entries(encryptedData)) { - const lookupKey = `${targetPath}/${dataKey}` - const fullPath = dataKeyToPath.get(lookupKey) - if (fullPath) { - set(spec, fullPath, value) - } - } - } -} - -/** - * Build a map from "groupPrefix/dataKey" → "full.schema.path". - * The dataKey is derived from the relative path by replacing dots with underscores, - * matching the convention used in buildSecretToNamespaceMap (sealed-secrets.ts). - */ -function buildDataKeyToPathMap(secretPaths: string[]): Map { - const result = new Map() - for (const secretPath of secretPaths) { - const groupPrefix = findGroupPrefix(secretPath) - if (!groupPrefix) continue - const relativePath = secretPath.slice(groupPrefix.length + 1) - if (!relativePath) continue - const dataKey = relativePath.replace(/\./g, '_') - result.set(`${groupPrefix}/${dataKey}`, secretPath) - } - return result -} - -/** - * Find the group prefix for a secret path — mirrors the logic in sealed-secrets.ts. - */ -function findGroupPrefix(secretPath: string): string | undefined { - const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) return `teamConfig.${teamMatch[1]}` - const appsMatch = secretPath.match(/^apps\.([^.]+)/) - if (appsMatch) return `apps.${appsMatch[1]}` - const [firstSegment] = secretPath.split('.') - if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment - return undefined -} - -function resolveSecretTargetPath(secretName: string, spec: Record): string | undefined { - // team-{name}-settings-secrets → teamConfig.{name} - const teamMatch = secretName.match(/^team-(.+)-settings-secrets$/) - if (teamMatch) return `teamConfig.${teamMatch[1]}` - - // {name}-secrets → apps.{name} or {name} - const nameMatch = secretName.match(/^(.+)-secrets$/) - if (!nameMatch) return undefined - - const [, name] = nameMatch - if (get(spec, `apps.${name}`) !== undefined) return `apps.${name}` - if (get(spec, name) !== undefined) return name - - return undefined -} - export function extractTeamDirectory(filePath: string): string { const match = filePath.match(/\/teams\/([^/]+)/) if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) diff --git a/tests/fixtures/env/apps/kubeflow-pipelines.yaml b/tests/fixtures/env/apps/kubeflow-pipelines.yaml index 81b27595ea..972dab0573 100644 --- a/tests/fixtures/env/apps/kubeflow-pipelines.yaml +++ b/tests/fixtures/env/apps/kubeflow-pipelines.yaml @@ -7,4 +7,5 @@ spec: persistence: mysql: size: 20Gi + rootPassword: mysqlsomesecretvalue _rawValues: {} diff --git a/tests/fixtures/env/settings/alerts.yaml b/tests/fixtures/env/settings/alerts.yaml index dbaee36fe9..f397bed631 100644 --- a/tests/fixtures/env/settings/alerts.yaml +++ b/tests/fixtures/env/settings/alerts.yaml @@ -6,5 +6,7 @@ spec: receivers: - slack - msteams - msteams: {} + msteams: + highPrio: https://xxxxxxx.com + lowPrio: https://xxxxxxxx.com slack: {} diff --git a/tests/fixtures/env/settings/obj.yaml b/tests/fixtures/env/settings/obj.yaml index f775be6591..b53d1a2c28 100644 --- a/tests/fixtures/env/settings/obj.yaml +++ b/tests/fixtures/env/settings/obj.yaml @@ -6,6 +6,7 @@ spec: provider: linode: accessKeyId: someaccessKeyId + secretAccessKey: somesecretvalue buckets: cnpg: my-clusterid-cnpg gitea: my-clusterid-gitea diff --git a/tests/fixtures/env/settings/otomi.yaml b/tests/fixtures/env/settings/otomi.yaml index b0ae2621dc..340e18e37b 100644 --- a/tests/fixtures/env/settings/otomi.yaml +++ b/tests/fixtures/env/settings/otomi.yaml @@ -5,6 +5,7 @@ metadata: spec: globalPullSecret: username: otomi + password: blablabla hasExternalDNS: true hasExternalIDP: false nodeSelector: From a9aa0485b3e294e57491272a35fbcfaf07e2dc11 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 14 Apr 2026 17:58:05 +0200 Subject: [PATCH 68/71] feat: enhance templates to handle secrets --- .../templates/deployment.yaml | 7 ++- charts/prometheus-msteams/values.yaml | 5 ++ helmfile.d/helmfile-10.monitoring.yaml.gotmpl | 18 +++++- .../fixtures/env/apps/kubeflow-pipelines.yaml | 1 - tests/fixtures/env/settings/alerts.yaml | 4 +- tests/fixtures/env/settings/obj.yaml | 1 - tests/fixtures/env/settings/otomi.yaml | 1 - values/k8s/k8s-raw-teams.gotmpl | 28 +++++++-- values/k8s/k8s-raw.gotmpl | 28 +++++++-- .../kubeflow-pipelines-raw.gotmpl | 61 ++++++++++++++----- .../prometheus-msteams-raw.gotmpl | 30 +++++++++ .../prometheus-msteams.gotmpl | 4 +- 12 files changed, 149 insertions(+), 39 deletions(-) create mode 100644 values/prometheus-msteams/prometheus-msteams-raw.gotmpl diff --git a/charts/prometheus-msteams/templates/deployment.yaml b/charts/prometheus-msteams/templates/deployment.yaml index f9d8b58d1f..a129997d48 100644 --- a/charts/prometheus-msteams/templates/deployment.yaml +++ b/charts/prometheus-msteams/templates/deployment.yaml @@ -41,7 +41,10 @@ spec: configMap: name: {{ template "app.name" . }}-card-template {{- if .Values.extraConfigmapMounts }} - {{ toYaml .Values.extraConfigmapMounts | nindent 8 }} + {{- toYaml .Values.extraConfigmapMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} @@ -69,7 +72,7 @@ spec: {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} args: - - -config-file=/etc/config/connectors.yaml + - -config-file={{ .Values.configFile }} - -template-file={{ .Values.templateFile }} {{- if .Values.workflowWebhook }} - -workflow-webhook={{ .Values.workflowWebhook }} diff --git a/charts/prometheus-msteams/values.yaml b/charts/prometheus-msteams/values.yaml index e7781c28af..f3e8b08cdb 100644 --- a/charts/prometheus-msteams/values.yaml +++ b/charts/prometheus-msteams/values.yaml @@ -67,12 +67,17 @@ customCardTemplate: "" templateFile: /etc/template/card.tmpl +configFile: /etc/config/connectors.yaml + extraConfigmapMounts: [] # extraConfigmapMounts: # - name: customConfigMap # configMap: # name: customConfigMapName +## Additional Volumes +extraVolumes: [] + ## Additional Volume mounts extraVolumeMounts: [] # extraVolumeMounts: diff --git a/helmfile.d/helmfile-10.monitoring.yaml.gotmpl b/helmfile.d/helmfile-10.monitoring.yaml.gotmpl index f5a7892b7f..3135fc6268 100644 --- a/helmfile.d/helmfile-10.monitoring.yaml.gotmpl +++ b/helmfile.d/helmfile-10.monitoring.yaml.gotmpl @@ -32,7 +32,23 @@ releases: installed: {{ and ($a | get "loki.enabled") (not ($a | get "loki.enableOpenTelemetry")) }} namespace: monitoring <<: *default + - name: prometheus-msteams-artifacts + installed: {{ has "msteams" ($v | get "alerts.receivers") }} + namespace: monitoring + <<: *raw - name: prometheus-msteams installed: {{ has "msteams" ($v | get "alerts.receivers") }} namespace: monitoring - <<: *default + missingFileHandler: Debug + chart: ../charts/prometheus-msteams + values: + - ../values/prometheus-msteams/prometheus-msteams.gotmpl + - snippets/common.gotmpl + - configFile: /etc/secret/connectors.yaml + extraVolumes: + - name: connectors-secret + secret: + secretName: prometheus-msteams-connectors + extraVolumeMounts: + - name: connectors-secret + mountPath: /etc/secret diff --git a/tests/fixtures/env/apps/kubeflow-pipelines.yaml b/tests/fixtures/env/apps/kubeflow-pipelines.yaml index 972dab0573..81b27595ea 100644 --- a/tests/fixtures/env/apps/kubeflow-pipelines.yaml +++ b/tests/fixtures/env/apps/kubeflow-pipelines.yaml @@ -7,5 +7,4 @@ spec: persistence: mysql: size: 20Gi - rootPassword: mysqlsomesecretvalue _rawValues: {} diff --git a/tests/fixtures/env/settings/alerts.yaml b/tests/fixtures/env/settings/alerts.yaml index f397bed631..dbaee36fe9 100644 --- a/tests/fixtures/env/settings/alerts.yaml +++ b/tests/fixtures/env/settings/alerts.yaml @@ -6,7 +6,5 @@ spec: receivers: - slack - msteams - msteams: - highPrio: https://xxxxxxx.com - lowPrio: https://xxxxxxxx.com + msteams: {} slack: {} diff --git a/tests/fixtures/env/settings/obj.yaml b/tests/fixtures/env/settings/obj.yaml index b53d1a2c28..f775be6591 100644 --- a/tests/fixtures/env/settings/obj.yaml +++ b/tests/fixtures/env/settings/obj.yaml @@ -6,7 +6,6 @@ spec: provider: linode: accessKeyId: someaccessKeyId - secretAccessKey: somesecretvalue buckets: cnpg: my-clusterid-cnpg gitea: my-clusterid-gitea diff --git a/tests/fixtures/env/settings/otomi.yaml b/tests/fixtures/env/settings/otomi.yaml index 340e18e37b..b0ae2621dc 100644 --- a/tests/fixtures/env/settings/otomi.yaml +++ b/tests/fixtures/env/settings/otomi.yaml @@ -5,7 +5,6 @@ metadata: spec: globalPullSecret: username: otomi - password: blablabla hasExternalDNS: true hasExternalIDP: false nodeSelector: diff --git a/values/k8s/k8s-raw-teams.gotmpl b/values/k8s/k8s-raw-teams.gotmpl index c647f9685f..83fdbd8fa0 100644 --- a/values/k8s/k8s-raw-teams.gotmpl +++ b/values/k8s/k8s-raw-teams.gotmpl @@ -1,6 +1,5 @@ {{- $v := .Values }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $dockerConfigTpl := readFile "../../helmfile.d/snippets/dockercfg.gotmpl" }} resources: {{- range $id, $team := omit $v.teamConfig "admin"}} {{- $ns := printf "team-%s" $id }} @@ -17,14 +16,31 @@ resources: istio-injection: enabled {{- end }} {{- with $v.otomi | get "globalPullSecret" nil }} - - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson + {{- $gpsUsername := . | get "username" "" }} + {{- $gpsServer := . | get "server" "docker.io" }} + {{- $gpsEmail := . | get "email" "not@val.id" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-pullsecret-global namespace: {{ $ns }} - data: - .dockerconfigjson: {{ tpl $dockerConfigTpl . | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-pullsecret-global + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' + data: + - secretKey: password + remoteRef: + key: otomi-secrets + property: globalPullSecret_password {{- end }} # patching service account here as helm does not recognize it as it's own - apiVersion: v1 diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 65631e6f26..f0e39c4c0b 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -1,6 +1,5 @@ {{- $v := .Values }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $dockerConfigTpl := readFile "../../helmfile.d/snippets/dockercfg.gotmpl" }} resources: {{- if ( (hasKey $v "bootstrap"))}} - {{- $v.bootstrap | toYaml | nindent 4 }} @@ -21,14 +20,31 @@ resources: {{- end }} {{- with $ns | get "labels" nil }}{{ toYaml . | nindent 8 }}{{ end }} {{- with $v.otomi | get "globalPullSecret" nil }} - - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson + {{- $gpsUsername := . | get "username" "" }} + {{- $gpsServer := . | get "server" "docker.io" }} + {{- $gpsEmail := . | get "email" "not@val.id" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-pullsecret-global namespace: {{ $ns.name }} - data: - .dockerconfigjson: {{ tpl $dockerConfigTpl . | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-pullsecret-global + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' + data: + - secretKey: password + remoteRef: + key: otomi-secrets + property: globalPullSecret_password {{- end }} - apiVersion: v1 kind: ServiceAccount diff --git a/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl b/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl index afdb5dd8dd..7ec645c839 100644 --- a/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl +++ b/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl @@ -1,29 +1,60 @@ {{- $v := .Values }} -{{- $kfp := index $v.apps "kubeflow-pipelines" }} {{- $obj := $v.obj.provider }} {{- $httpRoute := tpl (readFile "../../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} {{- $hostname := print "kubeflow-pipelines." $v.cluster.domainSuffix }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - labels: - app: kubeflow-pipelines name: kfp-mysql-secret - data: - password: "{{ $kfp.rootPassword | b64enc }}" - username: "{{ "root" | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: kfp-mysql-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: root + password: '{{ "{{ .rootPassword | toString }}" }}' + data: + - secretKey: rootPassword + remoteRef: + key: kubeflow-pipelines-secrets + property: rootPassword + {{- if eq $obj.type "linode" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: mlpipeline-obj-artifact + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: mlpipeline-obj-artifact + creationPolicy: Owner + template: + type: Opaque + data: + accesskey: {{ $obj.linode.accessKeyId }} + secretkey: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey + {{- else }} - apiVersion: v1 kind: Secret metadata: - labels: - app: kubeflow-pipelines name: mlpipeline-obj-artifact - data: - {{- if eq $obj.type "linode" }} - accesskey: "{{ $obj.linode.accessKeyId | b64enc }}" - secretkey: "{{ $obj.linode.secretAccessKey | b64enc }}" - {{- end }} + data: {} + {{- end }} - apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/values/prometheus-msteams/prometheus-msteams-raw.gotmpl b/values/prometheus-msteams/prometheus-msteams-raw.gotmpl new file mode 100644 index 0000000000..e105da2e33 --- /dev/null +++ b/values/prometheus-msteams/prometheus-msteams-raw.gotmpl @@ -0,0 +1,30 @@ +{{- $v := .Values }} +resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: prometheus-msteams-connectors + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: prometheus-msteams-connectors + creationPolicy: Owner + template: + type: Opaque + data: + connectors.yaml: | + connectors: + - high_priority_channel: {{ "{{ .highPrio | toString }}" }} + - low_priority_channel: {{ "{{ .lowPrio | toString }}" }} + data: + - secretKey: highPrio + remoteRef: + key: alerts-secrets + property: msteams_highPrio + - secretKey: lowPrio + remoteRef: + key: alerts-secrets + property: msteams_lowPrio diff --git a/values/prometheus-msteams/prometheus-msteams.gotmpl b/values/prometheus-msteams/prometheus-msteams.gotmpl index d766c25c40..d2158b3933 100644 --- a/values/prometheus-msteams/prometheus-msteams.gotmpl +++ b/values/prometheus-msteams/prometheus-msteams.gotmpl @@ -10,9 +10,7 @@ image: pullSecrets: - otomi-pullsecret-global {{- end }} -connectors: -- high_priority_channel: {{ $v.alerts.msteams.highPrio }} -- low_priority_channel: {{ $v.alerts.msteams.lowPrio }} +connectors: [] metrics: serviceMonitor: From 94d4947089839ae1e4008522239febb3f79a17f5 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 14 Apr 2026 21:48:50 +0200 Subject: [PATCH 69/71] fix: use static string for git type --- values/argocd/argocd-raw.gotmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index d400ea4a16..31853b9c64 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -29,7 +29,7 @@ resources: argocd.argoproj.io/secret-type: repo-creds type: Opaque data: - type: {{ print "git" | quote }} + type: git url: {{ printf "https://%s" $v._derived.giteaDomain }} username: {{ $v.otomi.git.username }} password: '{{ "{{ .gitPassword | toString }}" }}' @@ -57,7 +57,7 @@ resources: argocd.argoproj.io/secret-type: repo-creds type: Opaque data: - type: {{ print "git" | quote }} + type: git url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | quote }} username: {{ $v.otomi.git.username }} password: '{{ "{{ .gitPassword | toString }}" }}' @@ -86,7 +86,7 @@ resources: argocd.argoproj.io/secret-type: repo-creds type: Opaque data: - type: {{ print "git" | quote }} + type: git url: {{ $v.otomi.git.repoUrl }} username: {{ $v.otomi.git.username }} password: '{{ "{{ .gitPassword | toString }}" }}' From 51d20ed15968a55a902ee79738d1fa4fa3831324 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 14 Apr 2026 22:17:08 +0200 Subject: [PATCH 70/71] fix: simplify schema validation for secrets --- src/cmd/validate-values.ts | 47 ++++++++++++-------------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/cmd/validate-values.ts b/src/cmd/validate-values.ts index 47eecbe98b..c92100309b 100644 --- a/src/cmd/validate-values.ts +++ b/src/cmd/validate-values.ts @@ -1,5 +1,5 @@ import Ajv, { ValidateFunction } from 'ajv' -import { cloneDeep, unset } from 'lodash' +import { cloneDeep, difference, unset } from 'lodash' import { prepareEnvironment } from 'src/common/cli' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -19,44 +19,27 @@ const internalPaths: string[] = ['k8s', 'adminApps', 'teamApps'] */ export function removeSecretRequirements(schema: Record): Record { const cleaned = cloneDeep(schema) - removeSecretRequirementsInPlace(cleaned) + removeSecretsInPlace(cleaned) return cleaned } -function removeSecretRequirementsInPlace(schema: Record): void { - if (!schema || typeof schema !== 'object') return - - // Collect property names that have x-secret in this node - const secretProps = new Set() - if (schema.properties) { - for (const [propName, propSchema] of Object.entries(schema.properties as Record)) { - if (propSchema && typeof propSchema === 'object' && 'x-secret' in propSchema) { - secretProps.add(propName) - } - } - } - - // Remove secret properties from required array - if (Array.isArray(schema.required) && secretProps.size > 0) { - const filtered = schema.required.filter((r: string) => !secretProps.has(r)) - if (filtered.length === 0) { +function removeSecretsInPlace(node: any): void { + if (!node || typeof node !== 'object') return + if (node.properties && Array.isArray(node.required)) { + const secretProps = Object.keys(node.properties as Record).filter( + (key) => 'x-secret' in (node.properties[key] ?? {}), + ) + if (secretProps.length > 0) { + const filtered = difference(node.required as string[], secretProps) // eslint-disable-next-line no-param-reassign - delete schema.required - } else { + if (filtered.length === 0) delete node.required // eslint-disable-next-line no-param-reassign - schema.required = filtered + else node.required = filtered } } - - // Recurse into all sub-schemas - for (const value of Object.values(schema)) { - if (Array.isArray(value)) { - for (const item of value) { - removeSecretRequirementsInPlace(item) - } - } else if (value && typeof value === 'object') { - removeSecretRequirementsInPlace(value) - } + for (const value of Object.values(node as Record)) { + if (Array.isArray(value)) value.forEach(removeSecretsInPlace) + else if (value && typeof value === 'object') removeSecretsInPlace(value) } } From 7c41d61fd0cf28a4e410f9d4a337fc0fedd49a32 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 15 Apr 2026 18:12:35 +0200 Subject: [PATCH 71/71] feat: update argocd-redis-secret management and helm chart configuration --- src/cmd/install.test.ts | 2 ++ src/cmd/install.ts | 24 +++++++++---------- src/cmd/migrate.ts | 8 +++++-- tests/fixtures/env/apps/secrets.argocd.yaml | 6 ----- .../sealedsecrets/argocd-secrets.yaml | 16 +++++++++++++ values/argocd/argocd-raw.gotmpl | 24 +++++++++++++++---- 6 files changed, 54 insertions(+), 26 deletions(-) delete mode 100644 tests/fixtures/env/apps/secrets.argocd.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index ba2c8ce625..3162a9e9ba 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -20,6 +20,7 @@ jest.mock('src/common/k8s', () => ({ restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), + createArgoCdRedisSecret: jest.fn().mockResolvedValue(undefined), k8s: { app: jest.fn(), core: jest.fn(), @@ -37,6 +38,7 @@ jest.mock('src/common/hf', () => ({ hfValues: jest.fn(), deployEssential: jest.fn(), HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'], + HF_DEFAULT_SYNC_ON_INITIAL_INSTALL_ARGS: ['apply', '--include-needs'], })) jest.mock('src/common/git-config', () => ({ diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 03a9c5fa01..253260a2aa 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -195,6 +195,17 @@ export const installAll = async () => { d.info('No sealed secret manifests found, skipping controller restart') } + // Ensure ArgoCD Redis Secret exists and has Helm ownership metadata before Helm applies ArgoCD. + // redisPassword is an x-secret field and sealed in apl-secrets/argocd-secrets (decrypted just above), + // so we read it directly from K8s rather than from values. + d.info('Creating argocd-redis secret from sealed secret') + const argocdSealedSecret = await getK8sSecret('argocd-secrets', 'apl-secrets').catch(() => undefined) + await createArgoCdRedisSecret({ apps: { argocd: { redisPassword: argocdSealedSecret?.redisPassword } } }).catch( + (error) => { + d.warn('Could not pre-create argocd-redis secret:', getErrorMessage(error)) + }, + ) + // Deploy ESO (External Secrets Operator) d.info('Deploying external-secrets operator') await hf( @@ -321,18 +332,6 @@ const install = async (): Promise => { ) } -const prepareMandatorySecrets = async (): Promise => { - const d = terminal(`cmd:${cmdName}:prepareMandatorySecrets`) - d.info('Preparing mandatory secrets for installation') - - // Ensure ArgoCD Redis Secret exists and has Helm ownership metadata before Helm applies Argo CD. - d.info('Creating argocd-redis secret when possible') - const values = (await hfValues()) as Record - await createArgoCdRedisSecret(values).catch((error) => { - d.warn('Failed to create argocd-redis secret:', getErrorMessage(error)) - }) -} - export const module: CommandModule = { command: cmdName, describe: 'Install all k8s resources for first-time setup', @@ -341,7 +340,6 @@ export const module: CommandModule = { setParsedArgs(argv) setup() await prepareEnvironment() - await prepareMandatorySecrets() await install() }, } diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 269dd742c2..9ba975d460 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -1,6 +1,6 @@ import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' -import { randomUUID } from 'crypto' +import { randomBytes, randomUUID } from 'crypto' import { diff } from 'deep-diff' import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'fs' import { cp, rename as fsRename, mkdir, readFile, writeFile } from 'fs/promises' @@ -927,7 +927,11 @@ export const addRedisSecretForArgoCD = async (values: Record): Prom return } - await createArgoCdRedisSecret(values) + // redisPassword is an x-secret field: never present in values on disk. + // Generate one and write it into the shared values object so sopsMigration (v61) seals the same password. + const redisPassword = randomBytes(24).toString('base64url') + set(values, 'apps.argocd.redisPassword', redisPassword) + await createArgoCdRedisSecret({ apps: { argocd: { redisPassword } } }) // Components consume REDIS_PASSWORD as env var, so they must restart after secret rotation. const restartTargets = [ diff --git a/tests/fixtures/env/apps/secrets.argocd.yaml b/tests/fixtures/env/apps/secrets.argocd.yaml deleted file mode 100644 index aabb4c4109..0000000000 --- a/tests/fixtures/env/apps/secrets.argocd.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - redisPassword: somepassword -name: argocd-redis -metadata: - name: argocd diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml new file mode 100644 index 0000000000..e9e5f54146 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: argocd-secrets + namespace: apl-secrets +spec: + encryptedData: + redisPassword: someEncryptedPassword + template: + immutable: false + metadata: + name: argocd-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 9e7a2a8c70..9165539c0d 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -114,13 +114,27 @@ resources: remoteRef: key: keycloak-secrets property: idp_clientSecret - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - name: argocd-redis + name: argocd-redis-secret namespace: argocd - data: - auth: {{ $a.redisPassword | b64enc | quote }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-redis + creationPolicy: Merge + template: + data: + auth: '{{ "{{ .redisPassword | toString }}" }}' + data: + - secretKey: redisPassword + remoteRef: + key: argocd-secrets + property: redisPassword - apiVersion: v1 kind: Secret metadata: