diff --git a/.cspell.json b/.cspell.json index db8f8e9472..60b53049a1 100644 --- a/.cspell.json +++ b/.cspell.json @@ -103,6 +103,7 @@ "backoff", "basepath", "binzx", + "bitnami", "blackbox", "bootstrapper", "calico", diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b5d3d5980a..2ed881c70d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -91,7 +91,7 @@ on: options: - age - no_kms - default: age + default: no_kms certificate: type: choice description: Select certificate issuer diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index ba60b955ed..44e5a14fad 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -77,13 +77,9 @@ spec: value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }} - name: RECONCILE_INTERVAL_MS value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }} - {{- if hasKey $kms "sops" }} envFrom: - - secretRef: - name: apl-sops-secrets - secretRef: name: apl-git-credentials - {{- end }} volumeMounts: - name: otomi-values mountPath: /home/app/stack/env diff --git a/chart/apl/templates/sops-secrets.yaml b/chart/apl/templates/sops-secrets.yaml deleted file mode 100644 index 7544e0ec6c..0000000000 --- a/chart/apl/templates/sops-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- $kms := .Values.kms | default dict }} -{{- if hasKey $kms "sops" }} -{{- $v := $kms.sops }} -apiVersion: v1 -kind: Secret -metadata: - name: apl-sops-secrets - namespace: apl-operator -type: Opaque -data: -{{- with $v.azure }} - AZURE_CLIENT_ID: {{ .clientId | b64enc }} - AZURE_CLIENT_SECRET: {{ .clientSecret | b64enc }} -{{- with .tenantId }} - AZURE_TENANT_ID: {{ . | b64enc }}{{ end }} -{{- with .environment }} - AZURE_ENVIRONMENT: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.aws }} - AWS_ACCESS_KEY_ID: {{ .accessKey | b64enc }} - AWS_SECRET_ACCESS_KEY: {{ .secretKey | b64enc }} -{{- with .region }} - AWS_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.age }} - SOPS_AGE_KEY: {{ .privateKey | b64enc }} -{{- end }} -{{- with $v.google }} - GCLOUD_SERVICE_KEY: {{ .accountJson | b64enc }} -{{- with .project }} - GOOGLE_PROJECT: {{ . | b64enc }}{{ end }} -{{- with .region }} - GOOGLE_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/chart/chart-index/Chart.yaml b/chart/chart-index/Chart.yaml index 8424635397..6a4aa9c336 100644 --- a/chart/chart-index/Chart.yaml +++ b/chart/chart-index/Chart.yaml @@ -31,6 +31,9 @@ dependencies: - name: external-dns version: 1.20.0 repository: https://kubernetes-sigs.github.io/external-dns + - name: external-secrets + version: 0.14.3 + repository: https://charts.external-secrets.io - name: gitea version: 12.5.0 repository: https://dl.gitea.io/charts diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index 881ac3fd9d..0991e2e09b 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -44,6 +44,7 @@ spec: envFrom: - secretRef: name: apl-sops-secrets + optional: true - secretRef: name: apl-git-credentials livenessProbe: diff --git a/charts/external-secrets/Chart.yaml b/charts/external-secrets/Chart.yaml new file mode 100644 index 0000000000..f166cf473d --- /dev/null +++ b/charts/external-secrets/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +appVersion: 0.14.3 +description: External Secrets Operator for Kubernetes +home: https://external-secrets.io +keywords: +- secrets +- external-secrets +kubeVersion: '>=1.19.0-0' +maintainers: +- name: External Secrets Community + url: https://github.com/external-secrets/external-secrets +name: external-secrets +sources: +- https://github.com/external-secrets/external-secrets +type: application +version: 0.14.3 diff --git a/charts/external-secrets/crds/clusterexternalsecrets.yaml b/charts/external-secrets/crds/clusterexternalsecrets.yaml new file mode 100644 index 0000000000..bda5779fe9 --- /dev/null +++ b/charts/external-secrets/crds/clusterexternalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterexternalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterExternalSecret + listKind: ClusterExternalSecretList + plural: clusterexternalsecrets + shortNames: + - ces + singular: clusterexternalsecret + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.externalSecretSpec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshTime + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterExternalSecret creates ExternalSecrets across namespaces + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/clustersecretstores.yaml b/charts/external-secrets/crds/clustersecretstores.yaml new file mode 100644 index 0000000000..568ca452f1 --- /dev/null +++ b/charts/external-secrets/crds/clustersecretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustersecretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterSecretStore + listKind: ClusterSecretStoreList + plural: clustersecretstores + shortNames: + - css + singular: clustersecretstore + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterSecretStore represents a cluster-wide secret store + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/externalsecrets.yaml b/charts/external-secrets/crds/externalsecrets.yaml new file mode 100644 index 0000000000..07ba80b4e5 --- /dev/null +++ b/charts/external-secrets/crds/externalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ExternalSecret + listKind: ExternalSecretList + plural: externalsecrets + shortNames: + - es + singular: externalsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshInterval + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ExternalSecret reads secret data from external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/generatorstates.yaml b/charts/external-secrets/crds/generatorstates.yaml new file mode 100644 index 0000000000..0f50ec67aa --- /dev/null +++ b/charts/external-secrets/crds/generatorstates.yaml @@ -0,0 +1,39 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: generatorstates.generators.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: generators.external-secrets.io + names: + categories: + - externalsecrets + kind: GeneratorState + listKind: GeneratorStateList + plural: generatorstates + singular: generatorstate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GeneratorState tracks the state of generators + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/pushsecrets.yaml b/charts/external-secrets/crds/pushsecrets.yaml new file mode 100644 index 0000000000..7595969ca1 --- /dev/null +++ b/charts/external-secrets/crds/pushsecrets.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: pushsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: PushSecret + listKind: PushSecretList + plural: pushsecrets + singular: pushsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: PushSecret pushes secrets to external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/secretstores.yaml b/charts/external-secrets/crds/secretstores.yaml new file mode 100644 index 0000000000..a61ca3b805 --- /dev/null +++ b/charts/external-secrets/crds/secretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: secretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: SecretStore + listKind: SecretStoreList + plural: secretstores + shortNames: + - ss + singular: secretstore + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: SecretStore represents a source of secrets + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/templates/_helpers.tpl b/charts/external-secrets/templates/_helpers.tpl new file mode 100644 index 0000000000..d090bbc336 --- /dev/null +++ b/charts/external-secrets/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "external-secrets.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "external-secrets.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "external-secrets.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "external-secrets.selectorLabels" -}} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/external-secrets/templates/clusterrole.yaml b/charts/external-secrets/templates/clusterrole.yaml new file mode 100644 index 0000000000..daee3d4cd9 --- /dev/null +++ b/charts/external-secrets/templates/clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +rules: + - apiGroups: ["external-secrets.io"] + resources: ["externalsecrets", "externalsecrets/status", "externalsecrets/finalizers", "secretstores", "secretstores/status", "secretstores/finalizers", "clustersecretstores", "clustersecretstores/status", "clustersecretstores/finalizers", "clusterexternalsecrets", "clusterexternalsecrets/status", "clusterexternalsecrets/finalizers", "pushsecrets", "pushsecrets/status", "pushsecrets/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: ["generators.external-secrets.io"] + resources: ["generatorstates", "generatorstates/status", "generatorstates/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["secrets", "configmaps", "serviceaccounts", "serviceaccounts/token", "events"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "external-secrets.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} diff --git a/charts/external-secrets/templates/deployment.yaml b/charts/external-secrets/templates/deployment.yaml new file mode 100644 index 0000000000..2b2b4ea088 --- /dev/null +++ b/charts/external-secrets/templates/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "external-secrets.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "external-secrets.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "external-secrets.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + containers: + - name: external-secrets + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --concurrent=1 + - --metrics-addr=:8080 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/charts/external-secrets/templates/serviceaccount.yaml b/charts/external-secrets/templates/serviceaccount.yaml new file mode 100644 index 0000000000..c12da9edf8 --- /dev/null +++ b/charts/external-secrets/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/external-secrets/values.yaml b/charts/external-secrets/values.yaml new file mode 100644 index 0000000000..5f2abea1d5 --- /dev/null +++ b/charts/external-secrets/values.yaml @@ -0,0 +1,31 @@ +## External Secrets Operator values + +## @param replicaCount Number of ESO controller replicas +replicaCount: 1 + +## @param image ESO controller image configuration +image: + repository: ghcr.io/external-secrets/external-secrets + tag: v0.14.3 + pullPolicy: IfNotPresent + +## @param serviceAccount Service account configuration +serviceAccount: + create: true + name: external-secrets + +## @param installCRDs Install ESO CRDs +installCRDs: true + +## @param resources Resource limits and requests +resources: {} + +## @param webhook Webhook configuration +webhook: + replicaCount: 1 + resources: {} + +## @param certController Cert controller configuration +certController: + replicaCount: 1 + resources: {} diff --git a/charts/otomi-api/templates/deployment.yaml b/charts/otomi-api/templates/deployment.yaml index 4128ae7f26..aaf05d74f7 100644 --- a/charts/otomi-api/templates/deployment.yaml +++ b/charts/otomi-api/templates/deployment.yaml @@ -40,6 +40,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.gitConfig.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} livenessProbe: @@ -77,6 +81,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.gitConfig.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} ports: diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index d308ceec7f..e92b834e24 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -24,6 +24,9 @@ serviceAccount: imagePullSecrets: {} +gitConfig: + existingSecret: "" + rbac: # Specifies whether rbac should be set up create: true @@ -96,7 +99,6 @@ affinity: {} secrets: GIT_USER: GIT_EMAIL: - GIT_PASSWORD: env: GIT_REPO_URL: @@ -120,10 +122,3 @@ tools: # requests: # cpu: 100m # memory: 100Mi - - secrets: - SOPS_AGE_KEY: '' - GCLOUD_SERVICE_KEY: '' - AZURE_TENANT_ID: '' - AZURE_CLIENT_ID: '' - AZURE_CLIENT_SECRET: '' diff --git a/charts/prometheus-msteams/templates/deployment.yaml b/charts/prometheus-msteams/templates/deployment.yaml index f9d8b58d1f..a129997d48 100644 --- a/charts/prometheus-msteams/templates/deployment.yaml +++ b/charts/prometheus-msteams/templates/deployment.yaml @@ -41,7 +41,10 @@ spec: configMap: name: {{ template "app.name" . }}-card-template {{- if .Values.extraConfigmapMounts }} - {{ toYaml .Values.extraConfigmapMounts | nindent 8 }} + {{- toYaml .Values.extraConfigmapMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} @@ -69,7 +72,7 @@ spec: {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} args: - - -config-file=/etc/config/connectors.yaml + - -config-file={{ .Values.configFile }} - -template-file={{ .Values.templateFile }} {{- if .Values.workflowWebhook }} - -workflow-webhook={{ .Values.workflowWebhook }} diff --git a/charts/prometheus-msteams/values.yaml b/charts/prometheus-msteams/values.yaml index e7781c28af..f3e8b08cdb 100644 --- a/charts/prometheus-msteams/values.yaml +++ b/charts/prometheus-msteams/values.yaml @@ -67,12 +67,17 @@ customCardTemplate: "" templateFile: /etc/template/card.tmpl +configFile: /etc/config/connectors.yaml + extraConfigmapMounts: [] # extraConfigmapMounts: # - name: customConfigMap # configMap: # name: customConfigMapName +## Additional Volumes +extraVolumes: [] + ## Additional Volume mounts extraVolumeMounts: [] # extraVolumeMounts: diff --git a/core.yaml b/core.yaml index e47b8df0d3..82cd5c24cc 100644 --- a/core.yaml +++ b/core.yaml @@ -89,6 +89,11 @@ k8s: disablePolicyChecks: true - name: sealed-secrets app: sealed-secrets + - name: external-secrets + app: external-secrets + disableIstioInjection: true + - name: apl-secrets + disableIstioInjection: true - name: policy-reporter app: policy-reporter disablePolicyChecks: true diff --git a/helmfile.d/helmfile-01.init.yaml.gotmpl b/helmfile.d/helmfile-01.init.yaml.gotmpl index 6287a13d27..c31194a595 100644 --- a/helmfile.d/helmfile-01.init.yaml.gotmpl +++ b/helmfile.d/helmfile-01.init.yaml.gotmpl @@ -34,6 +34,14 @@ releases: installed: {{ $a | get "sealed-secrets.enabled" }} namespace: sealed-secrets <<: *default + - name: external-secrets + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *default + - name: external-secrets-artifacts + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *raw - name: cert-manager installed: true namespace: cert-manager diff --git a/helmfile.d/helmfile-04.init.yaml.gotmpl b/helmfile.d/helmfile-04.init.yaml.gotmpl index 18e61e13e2..b78d855bd8 100644 --- a/helmfile.d/helmfile-04.init.yaml.gotmpl +++ b/helmfile.d/helmfile-04.init.yaml.gotmpl @@ -25,6 +25,13 @@ releases: labels: pkg: apl-operator <<: *default + - name: apl-operator-artifacts + installed: true + namespace: apl-operator + labels: + pkg: apl-operator + app: core + <<: *raw - name: otomi-operator installed: true namespace: otomi-operator diff --git a/helmfile.d/helmfile-10.monitoring.yaml.gotmpl b/helmfile.d/helmfile-10.monitoring.yaml.gotmpl index f5a7892b7f..3135fc6268 100644 --- a/helmfile.d/helmfile-10.monitoring.yaml.gotmpl +++ b/helmfile.d/helmfile-10.monitoring.yaml.gotmpl @@ -32,7 +32,23 @@ releases: installed: {{ and ($a | get "loki.enabled") (not ($a | get "loki.enableOpenTelemetry")) }} namespace: monitoring <<: *default + - name: prometheus-msteams-artifacts + installed: {{ has "msteams" ($v | get "alerts.receivers") }} + namespace: monitoring + <<: *raw - name: prometheus-msteams installed: {{ has "msteams" ($v | get "alerts.receivers") }} namespace: monitoring - <<: *default + missingFileHandler: Debug + chart: ../charts/prometheus-msteams + values: + - ../values/prometheus-msteams/prometheus-msteams.gotmpl + - snippets/common.gotmpl + - configFile: /etc/secret/connectors.yaml + extraVolumes: + - name: connectors-secret + secret: + secretName: prometheus-msteams-connectors + extraVolumeMounts: + - name: connectors-secret + mountPath: /etc/secret diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index 8f576b8df7..767d1a3ce6 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -15,8 +15,6 @@ bases: {{- $tc := $v.teamConfig }} {{- $domain := ($v.cluster | get "domainSuffix" nil) }} {{- $gatewayName := $v.ingress.platformClass.className }} -{{- $slackTpl := tpl (readFile "../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} -{{- $opsgenieTpl := tpl (readFile "../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} {{- $httpRoute := tpl (readFile "../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} releases: {{- range $teamId, $team := omit $tc "admin" }} @@ -97,8 +95,8 @@ releases: podMetadata: labels: otomi.io/auth-policy: monitoring-{{ $teamId }} - # to do: load slackTpl and opsgenieTpl only if alerts.receicers = true - config: {{- tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 12 }} + useExistingSecret: true + configSecret: alertmanager-team-{{ $teamId }}-config route: main: hostnames: @@ -113,7 +111,10 @@ releases: prometheusSpec: {} grafana: enabled: {{ $teamSettings | get "managedMonitoring.grafana" false }} - adminPassword: {{ $teamSettings.password | quote }} + admin: + existingSecret: team-{{ $teamId }}-grafana-admin + userKey: admin-user + passwordKey: admin-password resources: {{- $teamApps.grafana.resources.grafana | toYaml | nindent 12 }} namespaceOverride: null # team-{{ $teamId }} @@ -165,7 +166,7 @@ releases: basicAuth: true basicAuthUser: {{ $teamId }} secureJsonData: - basicAuthPassword: {{ $teamSettings.password | quote }} + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} podLabels: otomi.io/auth-policy: monitoring-{{ $teamId }} extraManifests: @@ -220,4 +221,14 @@ releases: pipeline: otomi-task-teams values: - ../values/team-ns/team-ns.gotmpl + - name: team-secrets-{{ $teamId }} + installed: true + namespace: team-{{ $teamId }} + chart: ../charts/raw + labels: + tag: teams + team: {{ $teamId }} + pipeline: otomi-task-teams + values: + - ../values/team-secrets/team-secrets-raw.gotmpl {{- end }} diff --git a/helmfile.d/helmfile-70.shared.yaml.gotmpl b/helmfile.d/helmfile-70.shared.yaml.gotmpl index 2b0f64b250..6b25e08103 100644 --- a/helmfile.d/helmfile-70.shared.yaml.gotmpl +++ b/helmfile.d/helmfile-70.shared.yaml.gotmpl @@ -35,6 +35,13 @@ releases: pkg: oauth2-proxy app: core <<: *raw + - name: otomi-api-artifacts + installed: true + namespace: otomi + labels: + pkg: otomi + app: core + <<: *raw - name: otomi-api installed: true namespace: otomi diff --git a/helmfile.d/snippets/alertmanager-teams.gotmpl b/helmfile.d/snippets/alertmanager-teams.gotmpl index aefb5e30af..5f2a51ba8d 100644 --- a/helmfile.d/snippets/alertmanager-teams.gotmpl +++ b/helmfile.d/snippets/alertmanager-teams.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/alertmanager.gotmpl b/helmfile.d/snippets/alertmanager.gotmpl index a4eb2346bc..24448df09d 100644 --- a/helmfile.d/snippets/alertmanager.gotmpl +++ b/helmfile.d/snippets/alertmanager.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/defaults.gotmpl b/helmfile.d/snippets/defaults.gotmpl index a039aea1b9..fcccd8c114 100644 --- a/helmfile.d/snippets/defaults.gotmpl +++ b/helmfile.d/snippets/defaults.gotmpl @@ -23,10 +23,9 @@ environments: default: values: - apps: - argocd: - redisPassword: {{ randAlphaNum 32 }} - kubeflow-pipelines: - rootPassword: {{ randAlphaNum 32 }} + argocd: {} + kubeflow-pipelines: {} + gitea: {} {{- range $index, $ingressClassName := $ingressClassNames }} ingress-nginx-{{ $ingressClassName }}: {{- if eq $ingressClassName "platform" }} @@ -273,13 +272,9 @@ environments: useCloudShell: true downloadKubeconfig: true downloadDockerLogin: true - password: {{ randAlphaNum 32 }} {{- end }} {{- end }} - otomi: - adminPassword: {{ randAlphaNum 32 }} - git: - password: {{ randAlphaNum 20 }} + otomi: {} catalogs: default: name: default diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index bef4a5c457..b39c527848 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -994,6 +994,16 @@ environments: cpu: "2" memory: 1Gi _rawValues: {} + external-secrets: + resources: + operator: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 512Mi + _rawValues: {} rabbitmq: enabled: false resources: @@ -1190,4 +1200,4 @@ environments: aiEnabled: false users: [] versions: - specVersion: 60 + specVersion: 61 diff --git a/helmfile.d/snippets/derived.gotmpl b/helmfile.d/snippets/derived.gotmpl index 6c4344639e..9f5bf220bd 100644 --- a/helmfile.d/snippets/derived.gotmpl +++ b/helmfile.d/snippets/derived.gotmpl @@ -200,6 +200,7 @@ environments: ttyDomain: {{ print "tty." $domainSuffix }} consoleDomain: {{ printf "console.%s" $domainSuffix }} giteaDomain: {{ printf "gitea.%s" $domainSuffix }} + giteaDbUsername: gitea keycloakDomain: {{ printf "keycloak.%s" $domainSuffix }} harborDomain: {{ printf "harbor.%s" $domainSuffix }} tlsSecretName: {{ $tlsSecretName }} @@ -222,15 +223,10 @@ environments: {{- end }} external-dns: enabled: {{ $v.otomi.hasExternalDNS }} - harbor: - adminPassword: {{ $a | get "harbor.adminPassword" $v.otomi.adminPassword | quote }} - registry: - credentials: - password: {{ $a | get "harbor.registry.credentials.password" $v.otomi.adminPassword | quote }} + harbor: {} keycloak: enabled: true address: {{ $keycloakBaseUrl }} - adminPassword: {{ $a | get "keycloak.adminPassword" $v.otomi.adminPassword | quote }} ingress-nginx: enabled: true istio: diff --git a/helmfile.d/snippets/grafana.gotmpl b/helmfile.d/snippets/grafana.gotmpl index 9128cca891..95c64bebd7 100644 --- a/helmfile.d/snippets/grafana.gotmpl +++ b/helmfile.d/snippets/grafana.gotmpl @@ -9,8 +9,10 @@ analytics: org_role: Admin allow_sign_up: true oauth_auto_login: true - client_id: {{ .keycloak.clientID }} - client_secret: {{ .keycloak.clientSecret | quote }} + # $__env{} is Grafana's built-in environment variable interpolation for config values. + # See: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion + client_id: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_ID} + client_secret: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} scopes: email profile openid email_attribute_path: email login_attribute_path: username diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 24abce0e54..f58f91b84e 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -3,15 +3,15 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { bootstrap, - bootstrapSops, copyBasicFiles, createCustomCA, - getKmsValues, getStoredClusterSecrets, handleFileEntry, processValues, } from './bootstrap' +jest.mock('@linode/kubeseal-encrypt') + const { terminal } = stubs jest.mock('src/common/envalid', () => ({ @@ -36,23 +36,14 @@ describe('Bootstrapping values', () => { quiet: jest.fn(), }), }), - bootstrapSops: jest.fn(), + bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), ensureManifestDirectories: jest.fn(), handleFileEntry: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - existsSync: jest.fn(), - genSops: jest.fn(), - getDeploymentState: jest.fn().mockReturnValue({}), - getImageTagFromValues: jest.fn(), getK8sSecret: jest.fn(), - hfValues: jest.fn(), - isCli: true, migrate: jest.fn(), - nothrow: jest.fn(), pathExists: jest.fn(), processValues: jest.fn(), terminal, @@ -60,16 +51,14 @@ describe('Bootstrapping values', () => { } }) it('should call relevant sub routines', async () => { - deps.processValues.mockReturnValue(values) - deps.hfValues.mockReturnValue(values) + deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() - expect(deps.bootstrapSops).toHaveBeenCalled() + expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() }) it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { - deps.processValues.mockReturnValue(undefined) + deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) }) it('should get stored cluster secrets if those exist', async () => { deps.getK8sSecret.mockReturnValue({ 'otomi-generated-passwords': secrets }) @@ -82,329 +71,257 @@ describe('Bootstrapping values', () => { expect(res).toEqual(undefined) }) - describe('getKmsValues', () => { - let kmsValuesDeps: any - const ageKeys = { publicKey: 'agePublicKey', privateKey: 'agePrivateKey' } - const values = { someKey: 'someValue' } - beforeEach(() => { - kmsValuesDeps = { - generateAgeKeys: jest.fn().mockResolvedValue(ageKeys), - hfValues: jest.fn(), - } - }) - it('should not get kms values if those do not exist', async () => { - kmsValuesDeps.hfValues.mockReturnValue(values) - const res = await getKmsValues(kmsValuesDeps) - expect(res).toBeUndefined() + describe('Copying basic files', () => { + const deps = { + copy: jest.fn(), + copyFile: jest.fn(), + copySchema: jest.fn(), + mkdir: jest.fn(), + pathExists: jest.fn(), + terminal, + } + it('should not throw any exception', async () => { + const res = await copyBasicFiles(deps) + expect(res).toBe(undefined) }) - it('should get kms values if those exist', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'azure', - }, - }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'azure', - }, + }) + describe('Creating folders and files for workload', () => { + const values = { + values: { + image: { + repository: 'linode/apl-nodejs-helloworld', + tag: 'v1.5.1', }, - }) + }, + } + const workload = { + files: { + 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), + }, + } + const deps = { + loadYaml: jest.fn().mockReturnValue(workload), + mkdir: jest.fn(), + terminal, + writeFile: jest.fn(), + } + it('should create folders and files based on file entry in yaml', async () => { + await handleFileEntry(deps) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/teams/workloads/demo/values.yaml', JSON.stringify(values)) }) - it('should generate and return new age keys if provider is age and keys are missing', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'age', - }, + }) + describe('Checking for a custom CA', () => { + const deps = { + pki: { + rsa: { + generateKeyPair: jest.fn().mockReturnValue({ + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + }), }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, + createCertificate: jest.fn().mockReturnValue({ + publicKey: {}, + serialNumber: '01', + validity: {}, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + }), + certificateToPem: jest.fn(), + privateKeyToPem: jest.fn(), + } as unknown as typeof pki, + writeValues: jest.fn(), + terminal, + } + deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') + deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') + it('should create a new key pair when none exist', () => { + const res = createCustomCA(deps) + expect(res).toMatchObject({ + apps: { + 'cert-manager': { + customRootCA: 'certpem', + customRootCAKey: 'keypem', }, }, }) }) - describe('Copying basic files', () => { - const deps = { - copy: jest.fn(), - copyFile: jest.fn(), - copySchema: jest.fn(), - mkdir: jest.fn(), - pathExists: jest.fn(), + }) + describe('processing values', () => { + const generatedSecrets = { gen: 'x' } + const generatedPassword = 'generated-password' + const usersWithPasswords = [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: generatedPassword }, + ] + // Users stored directly in allSecrets (keycloak-operator derives groups from raw fields) + + const ca = { a: 'cert' } + const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) + const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) + const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) + let deps + beforeEach(() => { + deps = { + createCustomCA: jest.fn().mockReturnValue(ca), + createK8sSecret: jest.fn(), + generateSecrets: jest.fn().mockReturnValue(generatedSecrets), + getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), + loadYaml: jest.fn(), terminal, + writeValues: jest.fn(), + getUsers: jest.fn().mockReturnValue(usersWithPasswords), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), + stripAllSecrets: jest.fn().mockImplementation((v) => v), } - it('should not throw any exception', async () => { - const res = await copyBasicFiles(deps) - expect(res).toBe(undefined) - }) }) - describe('Creating folders and files for workload', () => { - const values = { - values: { - image: { - repository: 'linode/apl-nodejs-helloworld', - tag: 'v1.5.1', - }, - }, - } - const workload = { - files: { - 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), - }, - } - const deps = { - loadYaml: jest.fn().mockReturnValue(workload), - mkdir: jest.fn(), - terminal, - writeFile: jest.fn(), - } - it('should create folders and files based on file entry in yaml', async () => { - await handleFileEntry(deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) - expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/teams/workloads/demo/values.yaml', - JSON.stringify(values), - ) + describe('Creating CA', () => { + it('should ask to create a CA if issuer is custom-ca', async () => { + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalledTimes(1) }) }) - describe('Generating sops related files', () => { - const settings = { - kms: { - sops: { - provider: 'aws', - aws: { - keys: 'key1,key2', - }, - }, - }, - } - const deps = { - copyFile: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - gucci: jest.fn().mockReturnValue('ok'), - hfValues: jest.fn(), - loadYaml: jest.fn().mockReturnValue(settings), - pathExists: jest.fn(), - readFile: jest.fn(), - getKmsSettings: jest.fn(), - terminal, - writeFile: jest.fn(), - createUpdateGenericSecret: jest.fn(), - } - it('should create files on first run and en/de-crypt', async () => { - deps.pathExists.mockReturnValue(false) - deps.getKmsSettings.mockReturnValue({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, - }, - }, - }) - - await bootstrapSops(undefined, deps) - expect(deps.encrypt).toHaveBeenCalled() - expect(deps.decrypt).toHaveBeenCalled() + describe('processing app values', () => { + it('should generate secrets by taking values and previously generated secrets as input', async () => { + deps.loadYaml.mockReturnValue(values) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - it('should just create files on next runs', async () => { - deps.pathExists.mockReturnValue(true) - deps.hfValues.mockReturnValue(settings) - deps.decrypt = jest.fn() - deps.encrypt = jest.fn() - const res = await bootstrapSops(undefined, deps) - expect(res).toBe(undefined) - expect(deps.encrypt).not.toHaveBeenCalled() - expect(deps.decrypt).not.toHaveBeenCalled() + it('should overwrite a stored secret with one that was provided in values', async () => { + const newSecret = { secret: 'new' } + const valuesWithSecrets = merge(cloneDeep(values), newSecret) + const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) + deps.loadYaml.mockReturnValue(valuesWithSecrets) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(allSecrets) + await processValues(deps) + const expected = { ...allSecrets, users: usersWithPasswords } + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - }) - describe('Checking for a custom CA', () => { - const deps = { - pki: { - rsa: { - generateKeyPair: jest.fn().mockReturnValue({ - publicKey: { n: {}, e: {} }, - privateKey: { d: {}, p: {}, q: {} }, - }), - }, - createCertificate: jest.fn().mockReturnValue({ - publicKey: {}, - serialNumber: '01', - validity: {}, - sign: jest.fn(), - setSubject: jest.fn(), - setIssuer: jest.fn(), - setExtensions: jest.fn(), - }), - certificateToPem: jest.fn(), - privateKeyToPem: jest.fn(), - } as unknown as typeof pki, - writeValues: jest.fn(), - terminal, - } - deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') - deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') - it('should create a new key pair when none exist', () => { - const res = createCustomCA(deps) - expect(res).toMatchObject({ - apps: { - 'cert-manager': { - customRootCA: 'certpem', - customRootCAKey: 'keypem', - }, - }, - }) - }) - }) - describe('processing values', () => { - const generatedSecrets = { gen: 'x' } - const generatedPassword = 'generated-password' - const usersWithPasswords = [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: generatedPassword }, - ] - const ca = { a: 'cert' } - const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) - const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) - const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) - let deps - beforeEach(() => { - deps = { - createCustomCA: jest.fn().mockReturnValue(ca), - createK8sSecret: jest.fn(), - decrypt: jest.fn(), - existsSync: jest.fn(), - generateSecrets: jest.fn().mockReturnValue(generatedSecrets), - getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), - getKmsValues: jest.fn().mockReturnValue({}), - hfValues: jest.fn().mockReturnValue(values), - loadYaml: jest.fn(), - terminal, - validateValues: jest.fn().mockReturnValue(true), - writeValues: jest.fn(), - getUsers: jest.fn().mockReturnValue(usersWithPasswords), - generatePassword: jest.fn().mockReturnValue(generatedPassword), - addInitialPasswords: jest.fn().mockReturnValue(usersWithPasswords), - addPlatformAdmin: jest.fn().mockReturnValue(usersWithPasswords), - pathExists: jest.fn().mockReturnValue(true), - } + it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { + deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalled() }) - describe('Creating CA', () => { - it('should ask to create a CA if issuer is custom-ca', async () => { - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalledTimes(1) + it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { + deps.loadYaml.mockReturnValue({ + apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, }) + await processValues(deps) + expect(deps.createCustomCA).not.toHaveBeenCalled() }) - describe('processing app values', () => { - it('should not retrieve values from env dir', async () => { - await processValues(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) + it('should only store secrets', async () => { + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + await processValues(deps) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...mergedSecretsWithGenAndCa, + users: usersWithPasswords, }) - it('should generate secrets by taking values and previously generated secrets as input', async () => { - deps.loadYaml.mockReturnValue(values) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) - }) - it('should overwrite a stored secret with one that was provided in values', async () => { - const newSecret = { secret: 'new' } - const valuesWithSecrets = merge(cloneDeep(values), newSecret) - const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) - deps.loadYaml.mockReturnValue(valuesWithSecrets) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(allSecrets) - await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', allSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) - }) - it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { - deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalled() + }) + it('should not overwrite stored secrets', async () => { + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue({}) + deps.generateSecrets.mockReturnValue(generatedSecrets) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...generatedSecrets, + users: usersWithPasswords, }) - it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { - deps.loadYaml.mockReturnValue({ - apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, - }) - await processValues(deps) - expect(deps.createCustomCA).not.toHaveBeenCalled() + }) + it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { + deps.loadYaml.mockReturnValue({ + cluster: { name: 'bla', provider: 'dida' }, }) - it('should only store secrets', async () => { - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue(ca) - await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith( - 'otomi-generated-passwords', - 'otomi', - mergedSecretsWithGenAndCa, - ) + deps.getStoredClusterSecrets.mockReturnValue({ + users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) - it('should not overwrite stored secrets', async () => { - deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue({}) - deps.generateSecrets.mockReturnValue(generatedSecrets) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', generatedSecrets) + deps.generateSecrets.mockReturnValue({ gen: 'x' }) + deps.createCustomCA.mockReturnValue(ca) + const res = await processValues(deps) + // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, { + a: 'cert', + gen: 'x', + cluster: { name: 'bla', provider: 'dida' }, + users: [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, + ], }) - it('should only write and return original values', async () => { - deps.loadYaml.mockReturnValue({ - cluster: { name: 'bla', provider: 'dida' }, - }) - deps.createCustomCA.mockReturnValue({ a: 'cert' }) - deps.getStoredClusterSecrets.mockReturnValue({ - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) - deps.generateSecrets.mockReturnValue({ gen: 'x' }) - deps.createCustomCA.mockReturnValue(ca) - const res = await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, { - cluster: { name: 'bla', provider: 'dida' }, - a: 'cert', - gen: 'x', - users: [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: 'generated-password' }, - ], - }) - expect(res).toEqual({ - cluster: { name: 'bla', provider: 'dida' }, - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) + expect(res.originalInput).toEqual({ + cluster: { name: 'bla', provider: 'dida' }, + users: [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, + ], }) - it('should merge original with generated values and write them to env dir', async () => { - const writtenValues = merge( - cloneDeep(values), - cloneDeep(mergedSecretsWithGenAndCa), - cloneDeep({ users: usersWithPasswords }), - ) - deps.loadYaml.mockReturnValue({ ...values, users }) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.getUsers.mockReturnValue(usersWithPasswords) - await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, writtenValues) + }) + it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { + // mergedForDisk = merge(originalInput, allSecrets, { users }) + // allSecrets = merge(ca, storedSecrets, generatedSecrets) + users: usersWithPasswords + const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { + users: usersWithPasswords, }) + const expectedDiskValues = merge( + cloneDeep(secrets), + cloneDeep(values), + cloneDeep(allSecretsExpected), + cloneDeep({ users: usersWithPasswords }), + ) + deps.loadYaml.mockReturnValue({ ...values, users }) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.getUsers.mockReturnValue(usersWithPasswords) + await processValues(deps) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + }) + it('should call stripAllSecrets before writing values to disk', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) + await processValues(deps) + expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + }) + it('should still return full allSecrets for bootstrapSealedSecrets', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + const result = await processValues(deps) + // allSecrets should contain full unstripped secrets including pre-processed users + expect(result.allSecrets).toEqual( + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: usersWithPasswords }), + ) + }) + it('should store users as-is in allSecrets (keycloak-operator derives groups)', async () => { + const storedUsers = [ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + isPlatformAdmin: true, + teams: ['dev'], + }, + ] + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedUsers }) + deps.generateSecrets.mockReturnValue({}) + deps.createCustomCA.mockReturnValue({}) + deps.getUsers.mockReturnValue(storedUsers) + + const result = await processValues(deps) + + // Users stored directly — no groups transformation + expect(result.allSecrets.users).toEqual(storedUsers) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 63f5a0bd2d..d28a1fc749 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -1,31 +1,22 @@ import { randomUUID } from 'crypto' import { existsSync } from 'fs' -import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises' +import { copyFile, cp, mkdir, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' import { cloneDeep, get, merge, set } from 'lodash' import { pki } from 'node-forge' import path from 'path' import { bootstrapGit } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' -import { APL_OPERATOR_NS, DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' -import { decrypt, encrypt } from 'src/common/crypt' +import { DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' -import { - createK8sSecret, - createUpdateGenericSecret, - getDeploymentState, - getK8sSecret, - k8s, - secretId, -} from 'src/common/k8s' -import { getKmsSettings } from 'src/common/repo' +import { createK8sSecret, getK8sSecret, secretId } from 'src/common/k8s' +import { bootstrapSealedSecrets, stripAllSecrets } from 'src/common/sealed-secrets' import { ensureManifestDirectories, ensureTeamGitOpsDirectories, getFilename, - gucci, + getSchemaSecretsPaths, isCore, loadYaml, rootDir, @@ -35,116 +26,9 @@ import { BasicArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' import { $ } from 'zx' import { migrate } from './migrate' -import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -const kmsMap = { - aws: 'kms', - azure: 'azure_keyvault', - google: 'gcp_kms', - age: 'age', -} - -export const bootstrapSops = async ( - envDir = env.ENV_DIR, - deps = { - copyFile, - decrypt, - encrypt, - gucci, - loadYaml, - pathExists: existsSync, - getKmsSettings, - readFile, - terminal, - writeFile, - createUpdateGenericSecret, - }, -): Promise => { - const d = deps.terminal(`cmd:${cmdName}:genSops`) - const targetPath = `${envDir}/.sops.yaml` - const values = await deps.getKmsSettings(envDir) - - const provider: string | undefined = values?.kms?.sops?.provider - if (!provider) { - d.warn('No sops information given. Assuming no sops enc/decryption needed. Be careful!') - return - } - - const templatePath = `${rootDir}/tpl/.sops.yaml.gotmpl` - const kmsProvider = kmsMap[provider] as string - const kmsKeys = values.kms.sops[provider]?.keys as string - - const obj = { - provider: kmsProvider, - keys: kmsKeys, - } - - if (provider === 'age') { - const { publicKey } = values?.kms?.sops?.age ?? {} - let privateKey = values.kms?.sops?.age?.privateKey - if (privateKey?.startsWith('ENC')) { - privateKey = '' - } - obj.keys = publicKey - if (privateKey && !process.env.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(`${envDir}/.secrets`, `SOPS_AGE_KEY=${privateKey}`) - try { - await deps.createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', APL_OPERATOR_NS, { - SOPS_AGE_KEY: privateKey, - }) - } catch (e) { - d.warn('Failed to create or update apl-sops-secrets secret with SOPS_AGE_KEY, this might come later') - } - } - } - - const exists = deps.pathExists(targetPath) - d.log(`Creating sops file for provider ${provider}`) - const output = (await deps.gucci(templatePath, obj, true)) as string - await deps.writeFile(targetPath, output) - d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - - d.info('Copying sops related files') - // add sops related files - const file = '.gitattributes' - await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - - // prepare some credential files the first time and crypt some - if (!exists) { - if (isCli || env.OTOMI_DEV) { - // first time so we know we have values - const secretsFile = `${envDir}/.secrets` - d.log(`Creating secrets file: ${secretsFile}`) - if (provider === 'google') { - // and we also assume the correct values are given by using '!' (we want to err when not set) - const serviceKeyJson = JSON.parse(values.kms.sops!.google!.accountJson as string) - // and set it in env for later decryption - process.env.GCLOUD_SERVICE_KEY = values.kms.sops!.google!.accountJson - d.log('Creating gcp-key.json for vscode.') - await deps.writeFile(`${envDir}/gcp-key.json`, JSON.stringify(serviceKeyJson)) - d.log(`Creating credentials file: ${secretsFile}`) - await deps.writeFile(secretsFile, `GCLOUD_SERVICE_KEY=${JSON.stringify(JSON.stringify(serviceKeyJson))}`) - } else if (provider === 'aws') { - const v = values.kms.sops!.aws! - await deps.writeFile(secretsFile, `AWS_ACCESS_KEY_ID='${v.accessKey}'\nAWS_ACCESS_KEY_SECRET=${v.secretKey}`) - } else if (provider === 'azure') { - const v = values.kms.sops!.azure! - await deps.writeFile(secretsFile, `AZURE_CLIENT_ID='${v.clientId}'\nAZURE_CLIENT_SECRET=${v.clientSecret}`) - } else if (provider === 'age') { - const { privateKey } = values.kms.sops!.age! - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(secretsFile, `SOPS_AGE_KEY=${privateKey}`) - } - } - // now do a round of encryption and decryption to make sure we have all the files in place for later - await deps.encrypt(envDir) - await deps.decrypt(envDir) - } -} - export const copySchema = async (deps = { terminal, rootDir, env, isCore, loadYaml, copyFile }): Promise => { const d = deps.terminal(`cmd:${cmdName}:copySchema`) const { ENV_DIR } = env @@ -183,36 +67,6 @@ export const getStoredClusterSecrets = async ( return undefined } -export const generateAgeKeys = async (deps = { $, terminal }) => { - const d = deps.terminal(`cmd:${cmdName}:generateAgeKeys`) - try { - d.info('Generating age keys') - const result = await deps.$`age-keygen` - const { stdout } = result - const matchPublic = stdout?.match(/age[0-9a-z]+/) - const publicKey = matchPublic ? matchPublic[0] : '' - const matchPrivate = stdout?.match(/AGE-SECRET-KEY-[0-9A-Z]+/) - const privateKey = matchPrivate ? matchPrivate[0] : '' - const ageKeys = { publicKey, privateKey } - return ageKeys - } catch (error) { - d.log('Error generating age keys:', error) - throw error - } -} - -export const getKmsValues = async (values: Record, deps = { generateAgeKeys }) => { - const kms = values?.kms - if (!kms) return undefined - const provider = kms?.sops?.provider - if (!provider) return {} - if (provider !== 'age') return { kms } - const age = kms?.sops?.age - if (age?.publicKey && age?.privateKey) return { kms } - const ageKeys = await deps.generateAgeKeys() - return { kms: { sops: { provider: 'age', age: ageKeys } } } -} - export const addPlatformAdmin = (users: any[], domainSuffix: string) => { const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` const platformAdminExists = users.find((user) => user.email === defaultPlatformAdminEmail) @@ -299,22 +153,16 @@ export const processValues = async ( deps = { terminal, loadYaml, - decrypt, getStoredClusterSecrets, - getKmsValues, writeValues, - pathExists: existsSync, - hfValues, - validateValues, generateSecrets, createK8sSecret, createCustomCA, getUsers, - generatePassword, - addInitialPasswords, - addPlatformAdmin, + getSchemaSecretsPaths, + stripAllSecrets, }, -): Promise> => { +): Promise<{ originalInput: Record; allSecrets: Record }> => { const d = deps.terminal(`cmd:${cmdName}:processValues`) const { VALUES_INPUT } = env d.log(`Loading app values from ${VALUES_INPUT}`) @@ -331,23 +179,26 @@ export const processValues = async ( } else { caSecrets = deps.createCustomCA() } - // get any kms values & generate age keys if needed - const kmsValues = (await deps.getKmsValues(originalInput)) || {} // merge existing secrets over newly generated ones to keep them - const allSecrets = merge( - cloneDeep(caSecrets), - cloneDeep(storedSecrets), - cloneDeep(generatedSecrets), - cloneDeep(kmsValues), - ) + const allSecrets = merge(cloneDeep(caSecrets), cloneDeep(storedSecrets), cloneDeep(generatedSecrets)) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) - // we have generated all we need, now store everything by merging the original values over all the secrets - await deps.writeValues(merge(cloneDeep(allSecrets), cloneDeep(originalInput), cloneDeep({ users }))) + // Store users in allSecrets for sealed secret generation + // The keycloak-operator derives groups from isPlatformAdmin/isTeamAdmin/teams directly + allSecrets.users = users + // Include users in originalInput — getUsers() may return a detached array + // when originalInput had no 'users' key initially + const newInput = merge(cloneDeep(originalInput), cloneDeep({ users })) + // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets + // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) + const mergedForDisk = merge(cloneDeep(newInput), cloneDeep(allSecrets)) + const secretPaths = await deps.getSchemaSecretsPaths(Object.keys(get(mergedForDisk, 'teamConfig', {}))) + const valuesForDisk = deps.stripAllSecrets(mergedForDisk, secretPaths) + await deps.writeValues(valuesForDisk) // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return originalInput + return { originalInput: newInput, allSecrets } } // create file structure based on file entry @@ -435,16 +286,11 @@ export const createCustomCA = (deps = { terminal, pki, writeValues }): Record => { setParsedArgs(argv) await prepareEnvironment({ skipAllPreChecks: true }) - await decrypt() await bootstrap() await bootstrapGit() }, diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 076c14f19d..429eddb72d 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,23 +1,29 @@ import retry from 'async-retry' import { bootstrapGit, setIdentity } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' -import { APL_OPERATOR_NS } from 'src/common/constants' +import { + APL_OPERATOR_NS, + DEPLOYMENT_PASSWORDS_SECRET, + OTOMI_NAMESPACE, + OTOMI_SECRETS, + SEALED_SECRETS_NAMESPACE, +} from 'src/common/constants' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { getRepo, GitRepoConfig } from 'src/common/git-config' import { waitTillGitRepoAvailable } from 'src/common/gitea' import { hfValues } from 'src/common/hf' -import { createUpdateConfigMap, createUpdateGenericSecret, k8s } from 'src/common/k8s' +import { createUpdateConfigMap, createUpdateGenericSecret, getK8sSecret, k8s } from 'src/common/k8s' import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -export const rootDir = process.cwd() === '/home/app/stack/env' ? '/home/app/stack' : process.cwd() +const $git = $({ cwd: env.ENV_DIR }) interface Arguments extends HelmArguments { m?: string @@ -43,12 +49,11 @@ const isConflictError = (error: any): boolean => { const cleanupGitState = async (d: any): Promise => { try { - cd(env.ENV_DIR) // Try to abort any ongoing merge or rebase - await $`git merge --abort`.nothrow().quiet() - await $`git rebase --abort`.nothrow().quiet() + await $git`git merge --abort`.nothrow().quiet() + await $git`git rebase --abort`.nothrow().quiet() // Reset to the commit before our failed commit to discard local changes - await $`git reset --hard HEAD~1`.quiet() + await $git`git reset --hard HEAD~1`.quiet() d.info('Git state cleaned up after conflict - local commit discarded, reconciliation will retry') } catch (cleanupError) { d.warn('Error during git cleanup:', cleanupError?.message) @@ -64,52 +69,52 @@ const commitAndPush = async ( const d = terminal(`cmd:${cmdName}:commitAndPush`) d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' - const { password } = gitConfig ?? getRepo(values) - cd(env.ENV_DIR) + const { password } = gitConfig ?? (await getRepo(values)) try { try { - await $`git rev-list HEAD --count`.quiet() + await $git`git rev-list HEAD --count`.quiet() } catch { d.log('Very first commit') // We need at least two commits in repo, so git diff in Tekton pipeline always works. This is why the very first time we commit twice. - await $`git add README.md`.quiet() - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git add README.md`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } - await $`git add -A` + await $git`git add -A` // The below 'git status' command will always return at least single new line - const filesChangedCount = (await $`git status --untracked-files=no --porcelain`).toString().split('\n').length - 1 + const statusOutput = (await $git`git status --untracked-files=no --porcelain`).toString() + const filesChangedCount = statusOutput.split('\n').length - 1 if (filesChangedCount === 0) { d.log('Nothing to commit') return } - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } catch (e) { - d.log('commitAndPush error ', e?.message?.replace(password, '****')) - return + const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` + d.error(errorMsg) + throw new Error(errorMsg) } if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1' await retry( async () => { try { - cd(env.ENV_DIR) // Check if remote branch exists let remoteBranchExists = true try { - await $`git ls-remote --exit-code --heads origin ${branch}`.quiet() + await $git`git ls-remote --exit-code --heads origin ${branch}`.quiet() } catch { remoteBranchExists = false } // We're not always sure that we are on the correct branch, // so we checkout the branch and create it if it does not exist - await $`git checkout -B ${branch}`.quiet() + await $git`git checkout -B ${branch}`.quiet() if (remoteBranchExists) { - await $`git pull --rebase origin ${branch}`.quiet() + await $git`git pull --rebase origin ${branch}`.quiet() } else { d.log(`Remote branch '${branch}' does not exist. Skipping pull.`) } - await $`git push -u origin ${branch}`.quiet() + await $git`git push -u origin ${branch}`.quiet() } catch (pullPushError) { // Check if this is a merge conflict - if so, skip the commit if (isConflictError(pullPushError)) { @@ -143,16 +148,19 @@ export const commit = async ( await validateValues(overrideArgs) d.info('Preparing values') const values = (await hfValues()) as Record - // Use provided gitConfig if available (operator mode), otherwise read from values (bootstrap/install mode) - const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? getRepo(values) + const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? (await getRepo(values)) if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) + // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during + // the bootstrap phase before install) may have set the URL with unresolved placeholder + // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, + // we need to update the URL with the real credentials. + await $git`git remote set-url origin ${remote}`.nothrow().quiet() } else { - cd(env.ENV_DIR) - await setIdentity(username ?? 'otomi-admin', email) + await setIdentity(username ?? 'otomi-admin', email, env.ENV_DIR) // the url might need updating (e.g. if credentials changed) - await $`git remote set-url origin ${remote}` + await $git`git remote set-url origin ${remote}` } // let's wait until the remote is ready if (values?.apps!.gitea!.enabled ?? true) { @@ -164,26 +172,40 @@ export const commit = async ( } export async function initialSetupData(): Promise { + const d = terminal(`cmd:${cmdName}:initialSetupData`) const values = (await hfValues()) as Record const { domainSuffix } = values.cluster const { hasExternalIDP } = values.otomi - - const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` - const platformAdmin = values.users.find((user: any) => user.email === defaultPlatformAdminEmail) const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' - if (platformAdmin && !hasExternalIDP) { + if (!hasExternalIDP) { + // Read the platform admin's initialPassword from the generated passwords secret + let platformAdminPassword = '' + try { + const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, OTOMI_NAMESPACE) + const allSecrets = secretData?.[DEPLOYMENT_PASSWORDS_SECRET] + const users = allSecrets?.users || [] + const defaultEmail = `platform-admin@${domainSuffix}` + const platformAdmin = users.find((u: any) => u.email === defaultEmail) + platformAdminPassword = platformAdmin?.initialPassword || '' + } catch (error) { + d.warn(`Failed to read platform admin credentials: ${error instanceof Error ? error.message : error}`) + } return { domainSuffix, - username: platformAdmin.email, - password: platformAdmin.initialPassword, + username: `platform-admin@${domainSuffix}`, + password: platformAdminPassword, secretName, } } else { + // External IDP: show Keycloak admin credentials + const adminUsername = values?.apps?.keycloak?.adminUsername || 'otomi-admin' + const otomiSecret = await getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) + const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, - username: values.apps.keycloak.adminUsername, - password: values.apps.keycloak.adminPassword, + username: adminUsername, + password: adminPassword, secretName, } } diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index bbcae534cf..3162a9e9ba 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -19,6 +19,8 @@ jest.mock('src/common/k8s', () => ({ applyServerSide: jest.fn(), restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), + getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), + createArgoCdRedisSecret: jest.fn().mockResolvedValue(undefined), k8s: { app: jest.fn(), core: jest.fn(), @@ -36,15 +38,36 @@ jest.mock('src/common/hf', () => ({ hfValues: jest.fn(), deployEssential: jest.fn(), HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'], + HF_DEFAULT_SYNC_ON_INITIAL_INSTALL_ARGS: ['apply', '--include-needs'], })) jest.mock('src/common/git-config', () => ({ setGitConfig: jest.fn(), })) -jest.mock('zx', () => ({ - $: jest.fn(), - cd: jest.fn(), +jest.mock('zx', () => { + const mockResult = { exitCode: 0, stdout: '', stderr: '' } + const createMockProcessPromise = () => { + const promise = Promise.resolve(mockResult) + const chainable: any = promise + chainable.nothrow = jest.fn().mockReturnValue(chainable) + chainable.quiet = jest.fn().mockReturnValue(chainable) + return chainable + } + return { + $: jest.fn().mockImplementation(() => createMockProcessPromise()), + cd: jest.fn(), + } +}) + +jest.mock('src/common/sealed-secrets', () => ({ + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue([]), + restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock('src/common/utils', () => ({ + ...jest.requireActual('src/common/utils'), + rootDir: '/test/root', })) jest.mock('./commit', () => ({ @@ -67,11 +90,6 @@ jest.mock('src/common/cli', () => ({ prepareEnvironment: jest.fn(), })) -jest.mock('src/common/utils', () => ({ - ...jest.requireActual('src/common/utils'), - rootDir: '/test/root', -})) - jest.mock('src/common/yargs', () => ({ getParsedArgs: jest.fn().mockReturnValue({}), setParsedArgs: jest.fn(), @@ -118,7 +136,6 @@ describe('Install command', () => { stderr: '', }) mockDeps.deployEssential.mockResolvedValue(true) - mockDeps.$.mockResolvedValue(undefined) }) describe('module configuration', () => { @@ -264,32 +281,28 @@ describe('Install command', () => { }) describe('error handling', () => { - test('should handle deployment state errors', async () => { - const error = new Error('Failed to get deployment state') - mockDeps.getDeploymentState.mockRejectedValueOnce(error) + test('should throw on deployment state errors', async () => { + mockDeps.getDeploymentState.mockRejectedValueOnce(new Error('Failed to get deployment state')) - expect(mockDeps.getDeploymentState).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get deployment state') }) - test('should handle image tag retrieval errors', async () => { - const error = new Error('Failed to get image tag') - mockDeps.getImageTagFromValues.mockRejectedValueOnce(error) + test('should throw on image tag retrieval errors', async () => { + mockDeps.getImageTagFromValues.mockRejectedValueOnce(new Error('Failed to get image tag')) - expect(mockDeps.getImageTagFromValues).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get image tag') }) - test('should handle helmfile errors', async () => { - const error = new Error('Helmfile execution failed') - mockDeps.hf.mockRejectedValueOnce(error) + test('should throw on helmfile errors during sealed-secrets deploy', async () => { + mockDeps.hf.mockRejectedValueOnce(new Error('Helmfile execution failed')) - expect(mockDeps.hf).toBeDefined() + await expect(installAll()).rejects.toThrow('Helmfile execution failed') }) - test('should handle CRDs deployment errors', async () => { - const error = new Error('CRDs deployment failed') - mockDeps.applyServerSide.mockRejectedValueOnce(error) + test('should throw on essential deployment failure', async () => { + mockDeps.deployEssential.mockResolvedValueOnce(false) - expect(mockDeps.applyServerSide).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to deploy essential manifests') }) }) }) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 9baed7363b..253260a2aa 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -13,10 +13,16 @@ import { getDeploymentState, getHelmReleases, getK8sConfigMap, + getK8sSecret, k8s, setDeploymentState, waitForCRD, } from 'src/common/k8s' +import { + AppliedSecret, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -57,6 +63,68 @@ export const retryInstallStep = async ( ) } +/** + * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. + * Takes the list of applied secrets from applySealedSecretManifestsFromDir. + */ +const allSecretsExist = async (secrets: AppliedSecret[], deps = { getK8sSecret }): Promise => { + for (const { namespace, secretName } of secrets) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) return false + } catch { + return false + } + } + return true +} + +const waitForSealedSecrets = async ( + appliedSecrets: AppliedSecret[], + timeoutMs = env.SEALED_SECRETS_TIMEOUT_MS, + intervalMs = env.SEALED_SECRETS_INTERVAL_MS, + deps = { getK8sSecret, terminal }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) + + if (appliedSecrets.length === 0) { + d.info('No sealed secrets to wait for') + return + } + + d.info( + `Waiting for ${appliedSecrets.length} sealed secrets to be decrypted: ${appliedSecrets.map((s) => s.secretName).join(', ')}`, + ) + + await retry( + async () => { + const pending: string[] = [] + for (const { namespace, secretName } of appliedSecrets) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { + pending.push(`${namespace}/${secretName}`) + } + } + + if (pending.length > 0) { + throw new Error(`Sealed secrets not yet decrypted: ${pending.join(', ')}`) + } + + d.info('All sealed secrets have been decrypted') + }, + { + retries: Math.ceil(timeoutMs / intervalMs), + minTimeout: intervalMs, + maxTimeout: intervalMs, + factor: 1, + }, + ) +} + const getInitialInstallationMode = async (): Promise<'standard' | 'recovery'> => { const installationStatus = await getK8sConfigMap(APL_OPERATOR_NS, APL_OPERATOR_STATUS_CM, k8s.core()) const mode = installationStatus?.data?.installationMode @@ -91,16 +159,88 @@ export const installAll = async () => { throw new Error('Failed to deploy essential manifests') } + // Deploy sealed-secrets controller right after essentials + d.info('Deploying sealed-secrets controller') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=sealed-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for SealedSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') + + d.info('Applying SealedSecret manifests') + const appliedSecrets = await applySealedSecretManifestsFromDir(env.ENV_DIR) + + if (appliedSecrets.length > 0) { + // Check if all secrets are already decrypted (e.g. on retry after a previous successful run) + const allExist = await allSecretsExist(appliedSecrets, { getK8sSecret }) + if (allExist) { + d.info('All sealed secrets already decrypted, skipping controller restart') + } else { + // The controller may have started before the sealed-secrets-key TLS secret existed, + // causing it to generate its own key. Restarting forces it to pick up the pre-created key. + d.info('Restarting sealed-secrets controller to ensure correct key is used') + await restartSealedSecretsController() + + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets(appliedSecrets) + } + } else { + d.info('No sealed secret manifests found, skipping controller restart') + } + + // Ensure ArgoCD Redis Secret exists and has Helm ownership metadata before Helm applies ArgoCD. + // redisPassword is an x-secret field and sealed in apl-secrets/argocd-secrets (decrypted just above), + // so we read it directly from K8s rather than from values. + d.info('Creating argocd-redis secret from sealed secret') + const argocdSealedSecret = await getK8sSecret('argocd-secrets', 'apl-secrets').catch(() => undefined) + await createArgoCdRedisSecret({ apps: { argocd: { redisPassword: argocdSealedSecret?.redisPassword } } }).catch( + (error) => { + d.warn('Could not pre-create argocd-redis secret:', getErrorMessage(error)) + }, + ) + + // Deploy ESO (External Secrets Operator) + d.info('Deploying external-secrets operator') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for ExternalSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'externalsecrets.external-secrets.io') + + d.info('Deploying ESO ClusterSecretStore') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + // Deploy CRDs d.info('Deploying CRDs') await retryInstallStep(applyServerSide, 'charts/kube-prometheus-stack/charts/crds/crds') - // Wait for ServiceMonitor CRD to be established before deploying nginx await retryInstallStep(waitForCRD, 'servicemonitors.monitoring.coreos.com') await retryInstallStep(async () => $`kubectl apply -f charts/tekton-triggers/crds --server-side`) d.info('Deploying charts containing label stage=prep') await hf( { - // 'fileOpts' limits the hf scope and avoids parse errors (we only have basic values at this stage): fileOpts: 'helmfile.d/helmfile-02.init.yaml.gotmpl', labelOpts: ['stage=prep'], logLevel: logLevelString(), @@ -122,18 +262,42 @@ export const installAll = async () => { { streams: { stdout: d.stream.log, stderr: d.stream.error } }, ) + // Deploy cert-manager artifacts (ExternalSecrets, ClusterIssuers, Certificates) + // Must be after app=core (cert-manager CRDs) and after ESO + ClusterSecretStore + d.info('Deploying cert-manager artifacts') + await hf( + { + fileOpts: 'helmfile.d/helmfile-07.init.yaml.gotmpl', + labelOpts: ['name=cert-manager-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + if (!(env.isDev && env.DISABLE_SYNC)) { // Get the git configuration from values const values = (await hfValues()) as Record // Commit to Git repository await commit(true) + const gitBranch = values?.otomi?.git?.branch ?? 'main' await setGitConfig({ repoUrl: values?.otomi?.git?.repoUrl, - branch: values?.otomi?.git?.branch ?? 'main', + branch: gitBranch, email: values?.otomi?.git?.email, }) + // Verify the git push actually succeeded by checking the remote branch exists + d.info('Verifying git push succeeded') + const verifyResult = await $`git -C ${env.ENV_DIR} ls-remote --exit-code --heads origin ${gitBranch}` + .nothrow() + .quiet() + if (verifyResult.exitCode !== 0) { + throw new Error(`Git push verification failed: remote branch ${gitBranch} does not exist after commit`) + } + d.info('Git push verified successfully') + const initialData = await initialSetupData() await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password) await retryInstallStep(createWelcomeConfigMap, initialData.secretName, initialData.domainSuffix) @@ -168,18 +332,6 @@ const install = async (): Promise => { ) } -const prepareMandatorySecrets = async (): Promise => { - const d = terminal(`cmd:${cmdName}:prepareMandatorySecrets`) - d.info('Preparing mandatory secrets for installation') - - // Ensure ArgoCD Redis Secret exists and has Helm ownership metadata before Helm applies Argo CD. - d.info('Creating argocd-redis secret when possible') - const values = (await hfValues()) as Record - await createArgoCdRedisSecret(values).catch((error) => { - d.warn('Failed to create argocd-redis secret:', getErrorMessage(error)) - }) -} - export const module: CommandModule = { command: cmdName, describe: 'Install all k8s resources for first-time setup', @@ -188,7 +340,6 @@ export const module: CommandModule = { setParsedArgs(argv) setup() await prepareEnvironment() - await prepareMandatorySecrets() await install() }, } diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 0ff952ef26..b59ac980cf 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -6,6 +6,8 @@ import { getBuildName, policiesMigration, processDeletionEntry, + removeSopsArtifacts, + sopsMigration, } from 'src/cmd/migrate' import { terminal } from '../common/debug' import { env } from '../common/envalid' @@ -20,6 +22,7 @@ jest.mock('../common/k8s') jest.mock('../common/values') jest.mock('../common/yargs') jest.mock('../common/utils') +jest.mock('../common/sealed-secrets') jest.mock('zx') jest.mock('@linode/kubeseal-encrypt') jest.mock('fs', () => ({ @@ -912,6 +915,233 @@ describe('setDefaultAplCatalog migration', () => { }, 20000) }) +describe('sopsMigration', () => { + const mockTerminal = terminal + const mockExistsSync = jest.fn() + const mockGlobSync = jest.fn() + const mockGetExistingSealedSecretsCert = jest.fn() + const mockGetPemFromCertificate = jest.fn() + const mockGenerateSealedSecretsKeyPair = jest.fn() + const mockCreateSealedSecretsKeySecret = jest.fn() + const mockBuildSecretToNamespaceMap = jest.fn() + const mockCreateSealedSecretManifest = jest.fn() + const mockCreateUserSealedSecretManifests = jest.fn() + const mockWriteSealedSecretManifests = jest.fn() + const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) + const mockRestartSealedSecretsController = jest.fn().mockResolvedValue(undefined) + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) + const mockGetSchemaSecretsPaths = jest.fn() + const mockRemoveSopsArtifacts = jest.fn() + + const makeDeps = () => ({ + existsSync: mockExistsSync, + globSync: mockGlobSync, + terminal: mockTerminal, + getOrCreateSealedSecretsPem: jest.fn().mockImplementation(async (innerDeps: any) => { + const cert = await innerDeps.getExistingSealedSecretsCert() + if (cert) return innerDeps.getPemFromCertificate(cert) + const { certificate, privateKey } = innerDeps.generateSealedSecretsKeyPair() + await innerDeps.createSealedSecretsKeySecret(certificate, privateKey) + return innerDeps.getPemFromCertificate(certificate) + }), + getExistingSealedSecretsCert: mockGetExistingSealedSecretsCert, + getPemFromCertificate: mockGetPemFromCertificate, + generateSealedSecretsKeyPair: mockGenerateSealedSecretsKeyPair, + createSealedSecretsKeySecret: mockCreateSealedSecretsKeySecret, + buildSecretToNamespaceMap: mockBuildSecretToNamespaceMap, + createSealedSecretManifest: mockCreateSealedSecretManifest, + createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, + writeSealedSecretManifests: mockWriteSealedSecretManifests, + applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, + restartSealedSecretsController: mockRestartSealedSecretsController, + getK8sSecret: mockGetK8sSecret, + getSchemaSecretsPaths: mockGetSchemaSecretsPaths, + removeSopsArtifacts: mockRemoveSopsArtifacts, + }) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should skip when no .sops.yaml exists and no manifests on disk', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).not.toHaveBeenCalled() + }) + + it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should run full migration path', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue(undefined) + mockGenerateSealedSecretsKeyPair.mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem' }) + mockCreateSealedSecretsKeySecret.mockResolvedValue(undefined) + mockGetPemFromCertificate.mockReturnValue('spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([ + { namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { adminPassword: 'pass' } }, + ]) + mockCreateSealedSecretManifest.mockResolvedValue({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'gitea-secrets', namespace: 'apl-secrets', annotations: {} }, + spec: { encryptedData: { adminPassword: 'encrypted' }, template: {} }, + }) + mockCreateUserSealedSecretManifests.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword']) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + apps: { gitea: { adminPassword: 'pass' } }, + } + + await sopsMigration(values, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(mockBuildSecretToNamespaceMap).toHaveBeenCalled() + expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) + expect(mockWriteSealedSecretManifests).toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() + expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + // Secrets should be stripped from values (in-place mutation) + expect(values.apps.gitea.adminPassword).toBeUndefined() + }) + + it('should use existing certificate when available', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('existing-cert-pem') + mockGetPemFromCertificate.mockReturnValue('existing-spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).not.toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).not.toHaveBeenCalled() + expect(mockGetPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should handle users', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockCreateUserSealedSecretManifests.mockResolvedValue([ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'user1', namespace: 'apl-users' }, + }, + ]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + users: [{ name: 'user1', email: 'user1@example.com' }], + } + + await sopsMigration(values, makeDeps()) + + expect(mockCreateUserSealedSecretManifests).toHaveBeenCalledWith( + [{ name: 'user1', email: 'user1@example.com' }], + 'pem', + ) + }) + + it('should handle empty secrets gracefully', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockCreateSealedSecretManifest).not.toHaveBeenCalled() + expect(mockWriteSealedSecretManifests).toHaveBeenCalledWith([], env.ENV_DIR) + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + }) +}) + +describe('removeSopsArtifacts', () => { + it('should remove .sops.yaml, secrets files, and user files', () => { + const mockExistsSync = jest.fn().mockReturnValue(true) + const mockRmSync = jest.fn() + const mockGlobSync = jest + .fn() + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/users/some-uuid.yaml`]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/users/some-uuid.yaml`) + }) + + it('should skip .sops.yaml removal when it does not exist', () => { + const mockExistsSync = jest.fn().mockReturnValue(false) + const mockRmSync = jest.fn() + const mockGlobSync = jest.fn().mockReturnValue([]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).not.toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + }) +}) + describe('processDeletionEntry', () => { const mockDeleteFile = jest.fn() const deps = { deleteFile: mockDeleteFile } diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 50bc2f6e2a..9ba975d460 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -1,6 +1,6 @@ import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' -import { randomUUID } from 'crypto' +import { randomBytes, randomUUID } from 'crypto' import { diff } from 'deep-diff' import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'fs' import { cp, rename as fsRename, mkdir, readFile, writeFile } from 'fs/promises' @@ -20,7 +20,7 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd, sleep } from 'zx' -import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS } from '../common/constants' +import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from '../common/constants' import { createArgoCdRedisSecret, getArgoCdApp, @@ -31,8 +31,23 @@ import { restartStatefulSet, setArgoCdAppSync, } from '../common/k8s' +import { + applySealedSecretManifestsFromDir, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + createUserSealedSecretManifests, + generateSealedSecretsKeyPair, + getExistingSealedSecretsCert, + getOrCreateSealedSecretsPem, + getPemFromCertificate, + restartSealedSecretsController, + SealedSecretManifest, + writeSealedSecretManifests, +} from '../common/sealed-secrets' const cmdName = getFilename(__filename) +const sealedSecretManifestsGlob = `${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml` interface Arguments extends BasicArguments { dryRun?: boolean @@ -868,7 +883,12 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - await createCatalogSealedSecret(d, gitea as { adminUsername: string; adminPassword: string }) + const giteaSecrets = await getK8sSecret('gitea-secrets', SEALED_SECRETS_NAMESPACE) + const resolvedGitea = { + adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), + adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), + } + await createCatalogSealedSecret(d, resolvedGitea) secretCreated = true } catch (error) { d.error('Failed to create catalog sealed secret, continuing without it:', error) @@ -907,7 +927,11 @@ export const addRedisSecretForArgoCD = async (values: Record): Prom return } - await createArgoCdRedisSecret(values) + // redisPassword is an x-secret field: never present in values on disk. + // Generate one and write it into the shared values object so sopsMigration (v61) seals the same password. + const redisPassword = randomBytes(24).toString('base64url') + set(values, 'apps.argocd.redisPassword', redisPassword) + await createArgoCdRedisSecret({ apps: { argocd: { redisPassword } } }) // Components consume REDIS_PASSWORD as env var, so they must restart after secret rotation. const restartTargets = [ @@ -1012,6 +1036,143 @@ const addLinodeNBAnnotations = async (values: Record): Promise { + const d = deps.terminal(`cmd:${cmdName}:removeSopsArtifacts`) + + // Remove .sops.yaml — makes encrypt()/decrypt() no-ops + const sopsConfigPath = `${env.ENV_DIR}/.sops.yaml` + if (deps.existsSync(sopsConfigPath)) { + deps.rmSync(sopsConfigPath) + d.info('Removed .sops.yaml') + } + + // Remove SOPS-encrypted files + const sopsEncrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml`, { dot: false }) + for (const f of sopsEncrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } + + // Remove SOPS-decrypted files + const sopsDecrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml.dec`, { dot: false }) + for (const f of sopsDecrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } + + // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/namespaces/apl-users/sealedsecrets. + // These files may contain SOPS-encrypted data that was written by the "Write default values" step + // before SOPS decryption ran, contaminating the public YAML files with ENC[...] strings. + const userFiles = deps.globSync(`${env.ENV_DIR}/env/users/*.yaml`, { dot: false }) + for (const f of userFiles) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } +} + +export const sopsMigration = async ( + values: Record, + deps = { + existsSync, + globSync, + terminal, + getOrCreateSealedSecretsPem, + getExistingSealedSecretsCert, + getPemFromCertificate, + generateSealedSecretsKeyPair, + createSealedSecretsKeySecret, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createUserSealedSecretManifests, + writeSealedSecretManifests, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, + getK8sSecret, + getSchemaSecretsPaths, + removeSopsArtifacts, + }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:sopsMigration`) + + // Idempotency guard: no SOPS config means migration already ran. + // However, if SealedSecret manifests exist on disk but the K8s Secrets are not yet + // decrypted (e.g. the operator was killed after writing manifests but before applying + // them, or the controller used its auto-generated key), re-apply them and restart + // the controller so subsequent steps can resolve the git password. + if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { + const existingManifests = deps.globSync(sealedSecretManifestsGlob, { + dot: false, + }) + if (existingManifests.length > 0) { + try { + const platformSecret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) + if (!platformSecret) { + d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + await deps.restartSealedSecretsController() + } + } catch { + d.info('Could not reach K8s API to check secrets, skipping re-apply') + } + } + d.info('No .sops.yaml found, skipping SOPS migration') + return + } + + d.info('Starting SOPS to SealedSecrets migration') + + // Get or generate sealed-secrets key + const pem = await deps.getOrCreateSealedSecretsPem({ + terminal: deps.terminal, + getExistingSealedSecretsCert: deps.getExistingSealedSecretsCert, + getPemFromCertificate: deps.getPemFromCertificate, + generateSealedSecretsKeyPair: deps.generateSealedSecretsKeyPair, + createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, + }) + + // Build secret-to-namespace mappings + const teams = Object.keys((values.teamConfig as Record) || {}) + const mappings = await deps.buildSecretToNamespaceMap(values, teams, values) + + // Create core SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping) + manifests.push(manifest) + } + + // Create user SealedSecret manifests + const users = values.users as any[] | undefined + if (Array.isArray(users) && users.length > 0) { + const userManifests = await deps.createUserSealedSecretManifests(users, pem) + manifests.push(...userManifests) + } + + // Write manifests to disk + await deps.writeSealedSecretManifests(manifests, env.ENV_DIR) + d.info(`Wrote ${manifests.length} SealedSecret manifests`) + + // Apply SealedSecret manifests to the cluster so the sealed-secrets controller + // can decrypt them into K8s Secrets before the apply step needs them. + d.info('Applying SealedSecret manifests to cluster') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + + // Restart the sealed-secrets controller so it picks up the migration-generated key. + // Without this, the controller uses its auto-generated key and cannot decrypt. + d.info('Restarting sealed-secrets controller to use migration key') + await deps.restartSealedSecretsController() + + // Strip secrets from values (in-place mutation — writeValues() persists after return) + const secretPaths = await deps.getSchemaSecretsPaths(teams) + for (const path of secretPaths) { + unset(values, path) + } + + // Remove SOPS artifacts + deps.removeSopsArtifacts() + d.info('SOPS to SealedSecrets migration complete') +} + const setIngressDefault = async (values: Record) => { const d = terminal('setIngressDefault') if (values?.cluster?.provider !== 'linode') { @@ -1037,6 +1198,7 @@ const customMigrationFunctions: Record = { setDefaultAplCatalog, valkeyAndOauth2RedisPVCMigration, addLinodeNBAnnotations, + sopsMigration, setIngressDefault, addRedisSecretForArgoCD, } diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 550773d67a..b56ab13262 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -5,7 +5,7 @@ import { hfValues } from 'src/common/hf' import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { getRepo } from '../common/git-config' const cmdName = getFilename(__filename) @@ -17,13 +17,13 @@ export const pull = async (): Promise => { d.error('No values found, skipping git pull') return } - const gitRepo = getRepo(allValues) + const gitRepo = await getRepo(allValues) const { branch } = gitRepo d.info('Pulling latest values') - cd(env.ENV_DIR) + const $git = $({ cwd: env.ENV_DIR }) try { - await $`git fetch` - await $`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` + await $git`git fetch` + await $git`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` } catch (error) { d.warn( `An error occured when trying to pull (maybe not problematic).\nIf you see merge conflicts then please resolve these and run \`otomi commit\` again.`, diff --git a/src/cmd/validate-values.ts b/src/cmd/validate-values.ts index 63d604954b..c92100309b 100644 --- a/src/cmd/validate-values.ts +++ b/src/cmd/validate-values.ts @@ -1,5 +1,5 @@ import Ajv, { ValidateFunction } from 'ajv' -import { unset } from 'lodash' +import { cloneDeep, difference, unset } from 'lodash' import { prepareEnvironment } from 'src/common/cli' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -13,6 +13,36 @@ const cmdName = getFilename(__filename) const internalPaths: string[] = ['k8s', 'adminApps', 'teamApps'] +/** + * Remove x-secret properties from `required` arrays throughout the schema. + * Disk values have secrets stripped, so requiring them would always fail validation. + */ +export function removeSecretRequirements(schema: Record): Record { + const cleaned = cloneDeep(schema) + removeSecretsInPlace(cleaned) + return cleaned +} + +function removeSecretsInPlace(node: any): void { + if (!node || typeof node !== 'object') return + if (node.properties && Array.isArray(node.required)) { + const secretProps = Object.keys(node.properties as Record).filter( + (key) => 'x-secret' in (node.properties[key] ?? {}), + ) + if (secretProps.length > 0) { + const filtered = difference(node.required as string[], secretProps) + // eslint-disable-next-line no-param-reassign + if (filtered.length === 0) delete node.required + // eslint-disable-next-line no-param-reassign + else node.required = filtered + } + } + for (const value of Object.values(node as Record)) { + if (Array.isArray(value)) value.forEach(removeSecretsInPlace) + else if (value && typeof value === 'object') removeSecretsInPlace(value) + } +} + // TODO: Accept json path to validate - on empty, validate all export const validateValues = async (argv: HelmArguments = getParsedArgs(), envDir = env.ENV_DIR): Promise => { const d = terminal(`cmd:${cmdName}:validateValues`) @@ -33,12 +63,14 @@ export const validateValues = async (argv: HelmArguments = getParsedArgs(), envD d.info('Loading values-schema.yaml') const valuesSchema = (await loadYaml(`${rootDir}/values-schema.yaml`)) as Record + // Disk values have secrets stripped — remove x-secret fields from required arrays + const adjustedSchema = removeSecretRequirements(valuesSchema) d.debug('Initializing Ajv') const ajv = new Ajv({ allErrors: true, strict: false, strictTypes: false, verbose: true }) d.debug('Compiling Ajv validation') let validate: ValidateFunction try { - validate = ajv.compile(valuesSchema) + validate = ajv.compile(adjustedSchema) } catch (error) { throw new Error(`Schema is invalid: ${chalk.italic(error.message)}`) } diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 0e2a5a7d08..21868c55a2 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -1,18 +1,21 @@ import { existsSync } from 'fs' +import { get } from 'lodash' import { decrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' import { getRepo, GitRepoConfig } from 'src/common/git-config' import { hfValues } from 'src/common/hf' -import { getFilename } from 'src/common/utils' +import { stripAllSecrets } from 'src/common/sealed-secrets' +import { getFilename, getSchemaSecretsPaths } from 'src/common/utils' import { writeValues } from 'src/common/values' import { $, cd } from 'zx' const cmdName = getFilename(__filename) -export const setIdentity = async (username: string, email: string) => { - await $`git config --local user.name ${username}`.nothrow().quiet() - await $`git config --local user.email ${email}`.nothrow().quiet() +export const setIdentity = async (username: string, email: string, cwd?: string) => { + const $run = cwd ? $({ cwd }) : $ + await $run`git config --local user.name ${username}`.nothrow().quiet() + await $run`git config --local user.email ${email}`.nothrow().quiet() } export const recoverFromGit = async (gitConfig: GitRepoConfig): Promise => { @@ -36,7 +39,7 @@ export const bootstrapGit = async (inValues?: Record): Promise) - const { authenticatedUrl: remote, branch, email, username, password } = getRepo(values) + const { authenticatedUrl: remote, branch, email, username, password } = await getRepo(values) cd(env.ENV_DIR) if (existsSync(`${env.ENV_DIR}/.git`)) { d.info(`Git repo was already bootstrapped, setting identity just in case`) @@ -75,9 +78,12 @@ export const bootstrapGit = async (inValues?: Record): Promise + // Strip ALL secrets before writing to disk — secrets are in SealedSecrets only + const secretPaths = await getSchemaSecretsPaths(Object.keys(get(defaultValues, 'teamConfig', {}))) + const strippedValues = stripAllSecrets(defaultValues, secretPaths) // finally write back the new values without overwriting existing values d.info('Write default values to env repo') - await writeValues(defaultValues) + await writeValues(strippedValues) } if (!existsSync(`${env.ENV_DIR}/.git`)) { @@ -98,8 +104,10 @@ export const bootstrapGit = async (inValues?: Record): Promise(spec: { [K in keyof T]: ValidatorSpec }, options?: CleanOptions) { diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 994910bbef..78065af7b8 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -67,6 +67,15 @@ describe('git-config', () => { const result = await getGitCredentials() expect(result).toBeUndefined() }) + + it('should return undefined when password is a sealed-secret placeholder', async () => { + mockGetK8sSecret.mockResolvedValue({ + username: 'admin', + password: 'sealed:apl-secrets/otomi-secrets/git_password', + }) + const result = await getGitCredentials() + expect(result).toBeUndefined() + }) }) describe('getOldGitCredentials', () => { @@ -310,7 +319,7 @@ describe('git-config', () => { }) describe('getRepo', () => { - it('should use basic auth when username is provided', () => { + it('should use basic auth when username is provided', async () => { const values = { otomi: { git: { @@ -323,7 +332,7 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result).toEqual({ repoUrl: 'https://github.com/org/repo.git', authenticatedUrl: 'https://admin:s3cret@github.com/org/repo.git', @@ -334,7 +343,7 @@ describe('git-config', () => { }) }) - it('should use PAT auth (token only) when username is not provided', () => { + it('should use PAT auth (token only) when username is not provided', async () => { const values = { otomi: { git: { @@ -346,7 +355,7 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result).toEqual({ repoUrl: 'https://github.com/org/repo.git', authenticatedUrl: 'https://s3cret@github.com/org/repo.git', @@ -356,20 +365,19 @@ describe('git-config', () => { password: 's3cret', }) }) - - it('should throw when repoUrl is missing', () => { - expect(() => getRepo({ otomi: { git: {} } })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when repoUrl is missing', async () => { + await expect(getRepo({ otomi: { git: {} } })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when otomi.git is missing', () => { - expect(() => getRepo({ otomi: {} })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when otomi.git is missing', async () => { + await expect(getRepo({ otomi: {} })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when values is empty', () => { - expect(() => getRepo({})).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when values is empty', async () => { + await expect(getRepo({})).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should use GIT_REPO_URL env var in development mode', () => { + it('should use GIT_REPO_URL env var in development mode', async () => { process.env.NODE_ENV = 'development' process.env.GIT_REPO_URL = 'http://localhost:3000/dev/repo.git' @@ -385,9 +393,65 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result.repoUrl).toBe('http://localhost:3000/dev/repo.git') expect(result.authenticatedUrl).toBe('http://admin:s3cret@localhost:3000/dev/repo.git') }) + + it('should fallback to K8s secret when password is a sealed placeholder', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'real-password' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: 'sealed:apl-secrets/otomi-secrets/git_password', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(secretMock).toHaveBeenCalledWith('otomi-secrets', 'apl-secrets') + expect(result.password).toBe('real-password') + expect(result.authenticatedUrl).toContain('real-password') + }) + + it('should fallback to K8s secret when password is empty', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'from-k8s' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('from-k8s') + }) + + it('should keep empty password when K8s secret also has no password', async () => { + const secretMock = jest.fn().mockResolvedValue(null) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('') + }) }) }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 5d72436935..1fefc4fc48 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,7 +1,10 @@ import type { CoreV1Api } from '@kubernetes/client-node' -import { APL_OPERATOR_NS } from './constants' +import { APL_OPERATOR_NS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' +import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' +const d = terminal('common:git-config') + // Constants export const GIT_CONFIG_CONFIGMAP_NAME = 'apl-git-config' export const GIT_CONFIG_SECRET_NAME = 'apl-git-credentials' @@ -40,6 +43,11 @@ export async function getGitCredentials(): Promise { return undefined } + // Reject unresolved sealed-secret placeholders (e.g. during first deploy before secrets are decrypted) + if (typeof secretData.password === 'string' && secretData.password.startsWith('sealed:')) { + return undefined + } + return { username: secretData.username, password: secretData.password, @@ -118,7 +126,7 @@ export async function getStoredGitRepoConfig(): Promise { } const authenticatedUrl = url.toString() - return { repoUrl, authenticatedUrl, branch, email, username, password } + return { repoUrl, authenticatedUrl, branch, email, username, password } as GitRepoConfig } /** @@ -138,8 +146,10 @@ export async function setGitConfig(config: Partial, coreV1Api?: C /** * Gets repository configuration from values, constructing the authenticated URL with embedded credentials. + * If password is missing or is an unresolved sealed-secret placeholder, falls back to reading + * the real password from the K8s secret (populated by ESO from SealedSecrets). */ -export const getRepo = (values: Record): GitRepoConfig => { +export const getRepo = async (values: Record, deps = { getK8sSecret }): Promise => { const otomiGit = values?.otomi?.git if (!otomiGit?.repoUrl) { @@ -149,10 +159,31 @@ export const getRepo = (values: Record): GitRepoConfig => { otomiGit.repoUrl = process.env.GIT_REPO_URL } const username: string | undefined = otomiGit?.username - const password = otomiGit?.password + let password = '' const email = otomiGit?.email const branch = otomiGit?.branch + // Always try the K8s secret first for the real password. + // On disk, secrets are stripped — the values spec has no plaintext password. + try { + const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) + if (secret?.git_password) { + password = String(secret.git_password) + d.debug('Read git password from K8s secret (ESO)') + } + } catch { + d.debug('Could not read git password from K8s secret') + } + + // Fall back to values if K8s secret is not available (e.g., during bootstrap) + if (!password) { + const valuesPassword = otomiGit?.password ?? '' + // Only use the values password if it's not an unresolved sealed-secret placeholder + if (valuesPassword && !(typeof valuesPassword === 'string' && valuesPassword.startsWith('sealed:'))) { + password = valuesPassword + } + } + const repoUrl = otomiGit?.repoUrl as string const url = new URL(repoUrl) if (username) { @@ -164,5 +195,5 @@ export const getRepo = (values: Record): GitRepoConfig => { } const authenticatedUrl = url.toString() - return { repoUrl, authenticatedUrl, branch, email, username, password } + return { repoUrl, authenticatedUrl, branch, email, username, password } as GitRepoConfig } diff --git a/src/common/gitea.ts b/src/common/gitea.ts index ab6268419b..5d46cbdafd 100644 --- a/src/common/gitea.ts +++ b/src/common/gitea.ts @@ -1,5 +1,5 @@ import retry from 'async-retry' -import { $, cd } from 'zx' +import { $ } from 'zx' import { APL_OPERATOR_NS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -29,14 +29,14 @@ export const waitTillGitRepoAvailable = async (repoUrl: string): Promise = const d = terminal('common:gitea:waitTillGitRepoAvailable') await retry( async () => { + const $git = $({ cwd: env.ENV_DIR }) try { - cd(env.ENV_DIR) // the ls-remote exists with zero even if repo is empty - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } catch (e) { if (e.stderr && e.stderr.includes('remote: Update your password')) { await resetGiteaPasswordValidity() - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } else { d.warn(`The values repository is not yet reachable. Retrying in ${env.MIN_TIMEOUT} ms`) throw e diff --git a/src/common/hf.ts b/src/common/hf.ts index 5b8a5354b2..957bcf5c00 100644 --- a/src/common/hf.ts +++ b/src/common/hf.ts @@ -72,10 +72,12 @@ const hfCore = (args: HFParams, envDir = env.ENV_DIR): ProcessPromise => { stringArray.push(`--log-level=${paramsCopy.logLevel.toLowerCase()}`) process.env.HELM_DIFF_COLOR = 'true' process.env.HELM_DIFF_USE_UPGRADE_DRY_RUN = 'true' + // Always run helmfile from rootDir to ensure helmfile.d/ is found regardless of process cwd + const $hf = $({ cwd: rootDir }) if ((parsedArgs?.dryRun || parsedArgs?.local) && paramsCopy.args.includes('sync')) { - return $`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } else { - return $`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } } diff --git a/src/common/k8s.test.ts b/src/common/k8s.test.ts index bc35c28693..99d454b1d4 100644 --- a/src/common/k8s.test.ts +++ b/src/common/k8s.test.ts @@ -14,8 +14,8 @@ import { V1StatefulSet, V1Status, } from '@kubernetes/client-node' -import { X509Certificate } from 'crypto' import retry from 'async-retry' +import { X509Certificate } from 'crypto' import { ARGOCD_APP_PARAMS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -733,8 +733,8 @@ describe('helm operations in progress check', () => { await k8s.deletePendingHelmReleases() expect(mockGetPendingHelmReleases).toHaveBeenCalled() - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(1, 'release-1', 'ns-1') - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2') + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(2, 'release-1', 'ns-1', 2) + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2', 1) }) }) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index 263b6728c8..a97bed32b6 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -156,12 +156,12 @@ export const getK8sSecret = async (name: string, namespace: string): Promise { +export const deleteSecretForHelmRelease = async (releaseName: string, namespace: string, revision = 1) => { const d = terminal('common:k8s:deleteSecretForHelmRelease') - d.info(`Deleting secret for Helm release ${releaseName} in namespace ${namespace}`) + d.info(`Deleting secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) try { - await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v1`, namespace }) - d.debug(`Deleted secret for Helm release ${releaseName} in namespace ${namespace}`) + await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v${revision}`, namespace }) + d.debug(`Deleted secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) } catch (error) { if (error?.response?.statusCode !== 404) { throw error @@ -201,9 +201,11 @@ export const deletePendingHelmReleases = async (): Promise => { const d = terminal(`common:k8s:deletePendingHelmReleases`) const pendingHelmReleases = await getPendingHelmReleases() if (pendingHelmReleases.length > 0) { - d.info(`Pending Helm operations detected for releases: ${pendingHelmReleases.join(', ')}. removing secrets...`) + d.info( + `Pending Helm operations detected for releases: ${pendingHelmReleases.map((r) => `${r.namespace}/${r.name}:v${r.revision}`).join(', ')}. removing secrets...`, + ) for (const release of pendingHelmReleases) { - await deleteSecretForHelmRelease(release.name, release.namespace) + await deleteSecretForHelmRelease(release.name, release.namespace, release.revision) } } } @@ -1071,6 +1073,33 @@ export async function waitForCRD(crdName: string, timeoutSeconds: number = 60): } } +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string): Promise => { + const d = terminal(`common:k8s:ensureNamespaceExists`) + + try { + await k8s.core().readNamespace({ name: namespace }) + d.debug(`Namespace ${namespace} already exists`) + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info(`Creating namespace ${namespace}`) + await k8s.core().createNamespace({ + body: { + metadata: { + name: namespace, + labels: { name: namespace }, + }, + }, + }) + } else { + throw error + } + } +} + export async function getSealedSecretsPEM(): Promise { const d = terminal('common:k8s:getSealedSecretsPEM') const namespace = 'sealed-secrets' diff --git a/src/common/repo.test.ts b/src/common/repo.test.ts index de6c443315..5c950477b9 100644 --- a/src/common/repo.test.ts +++ b/src/common/repo.test.ts @@ -10,7 +10,6 @@ import { getUniqueIdentifierFromFilePath, hasCorrespondingDecryptedFile, renderManifest, - renderManifestForSecrets, saveResourceGroupToFiles, sortTeamConfigArraysByName, sortUserArraysByName, @@ -47,7 +46,6 @@ describe('getFilePath', () => { const data = {} const jsonPath = ['$', 'apps', 'grafana'] expect(getFilePath(fileMap, jsonPath, data, '')).toEqual('/tmp/values/env/apps/grafana.yaml') - expect(getFilePath(fileMap, jsonPath, data, 'secrets.')).toEqual('/tmp/values/env/apps/secrets.grafana.yaml') }) it('should get path for teamA', () => { const fileMap: FileMap = { @@ -234,11 +232,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'teamConfig', 'demo', 'netpols', '[1]'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/teams/demo/netpols/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/teams/demo/netpols/secrets.a.yaml') }) it('should return file path for platfrom dns', () => { @@ -254,11 +249,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/settings/dns.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/settings/secrets.dns.yaml') }) it('should return file path for user', () => { const fileMap: FileMap = { @@ -273,11 +265,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { id: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/users/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/users/secrets.a.yaml') }) }) @@ -692,13 +681,6 @@ describe('AplCatalog', () => { expect(filePath).toBe('/tmp/values/env/catalogs/default.yaml') }) - it('should return the correct secrets file path for a catalog', () => { - const data = { name: 'default', repositoryUrl: 'https://example.com/charts.git' } - const jsonPath = ['$', 'catalogs', 'default'] - const filePath = getFilePath(catalogFileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/values/env/catalogs/secrets.default.yaml') - }) - it('should use the map key for the file name, not data.name', () => { const data = { name: 'production-charts', repositoryUrl: 'https://example.com/charts.git' } const jsonPath = ['$', 'catalogs', 'prod'] @@ -779,17 +761,6 @@ describe('AplCatalog', () => { }) }) - describe('renderManifestForSecrets', () => { - it('should render a secrets manifest for a catalog', () => { - const data = { secretName: 'git-credentials' } - const manifest = renderManifestForSecrets(catalogFileMap, 'default', data) - - expect(manifest.kind).toBe('AplCatalog') - expect(manifest.metadata.name).toBe('default') - expect(manifest.spec).toEqual(data) - }) - }) - describe('saveResourceGroupToFiles', () => { it('should save a single catalog to a file', async () => { const writeValuesToFile = jest.fn() @@ -856,39 +827,6 @@ describe('AplCatalog', () => { ) }) - it('should save catalog secrets to files with secrets prefix', async () => { - const writeValuesToFile = jest.fn() - const valuesPublic = { - catalogs: { - default: { - name: 'default', - repositoryUrl: 'https://github.com/linode/apl-charts.git', - branch: 'main', - enabled: true, - }, - }, - } - const valuesSecrets = { - catalogs: { - default: { - secretName: 'git-credentials', - }, - }, - } - - await saveResourceGroupToFiles(catalogFileMap, valuesPublic, valuesSecrets, { writeValuesToFile }) - - expect(writeValuesToFile).toHaveBeenCalledTimes(2) - expect(writeValuesToFile).toHaveBeenCalledWith( - '/tmp/values/env/catalogs/secrets.default.yaml', - expect.objectContaining({ - kind: 'AplCatalog', - metadata: { name: 'default' }, - spec: valuesSecrets.catalogs.default, - }), - ) - }) - it('should not write anything when there are no catalogs', async () => { const writeValuesToFile = jest.fn() diff --git a/src/common/repo.ts b/src/common/repo.ts index f37993ca7b..ab89e6060b 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -462,28 +462,13 @@ export function renderManifest( return manifest } -export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record) { - let spec = data - if (fileMap.resourceGroup === 'users') { - spec = omit(data, ['id', 'name']) - } - return { - kind: fileMap.kind, - metadata: { - name: resourceName, - }, - spec, - } -} - export async function saveResourceGroupToFiles( fileMap: FileMap, valuesPublic: Record, - valuesSecrets: Record, + _valuesSecrets: Record, deps = { writeValuesToFile }, ): Promise { const jsonPathsValuesPublic = jsonpath.nodes(valuesPublic, fileMap.jsonPathExpression) - const jsonPathsvaluesSecrets = jsonpath.nodes(valuesSecrets, fileMap.jsonPathExpression) await Promise.all( jsonPathsValuesPublic.map(async (node) => { @@ -506,22 +491,8 @@ export async function saveResourceGroupToFiles( }), ) - await Promise.all( - jsonPathsvaluesSecrets.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.') - const resourceName = getResourceName(fileMap, nodePath, nodeValue) - const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue) - await deps.writeValuesToFile(filePath, manifest) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) + // Secrets are now stored as SealedSecret manifests via buildSecretToNamespaceMap() + writeSealedSecretManifests() + // No longer writing secrets.*.yaml files } export function getUniqueIdentifierFromFilePath(filePath: string): string { @@ -571,7 +542,6 @@ export async function setValuesFile( deps = { pathExists: existsSync, loadValues, writeFile }, ): Promise { const valuesPath = path.join(envDir, 'values-repo.yaml') - // if (await deps.pathExists(valuesPath)) return valuesPath const allValues = await deps.loadValues(envDir) await deps.writeFile(valuesPath, objectToYaml(allValues)) return valuesPath diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts new file mode 100644 index 0000000000..69dd2c6e96 --- /dev/null +++ b/src/common/sealed-secrets.test.ts @@ -0,0 +1,643 @@ +import { pki } from 'node-forge' +import stubs from 'src/test-stubs' +import { + applySealedSecretManifests, + bootstrapSealedSecrets, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + createUserSealedSecretManifests, + generateSealedSecretsKeyPair, + getPemFromCertificate, + restartSealedSecretsController, + SealedSecretManifest, + stripAllSecrets, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('src/common/k8s', () => ({ + getK8sSecret: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), + b64enc: jest.fn((v: string) => Buffer.from(v).toString('base64')), + k8s: { + core: jest.fn().mockReturnValue({ + createNamespacedSecret: jest.fn().mockResolvedValue({}), + }), + app: jest.fn().mockReturnValue({ + patchNamespacedDeployment: jest.fn().mockResolvedValue({}), + readNamespacedDeployment: jest + .fn() + .mockResolvedValue({ spec: { replicas: 1 }, status: { updatedReplicas: 1, availableReplicas: 1 } }), + }), + custom: jest.fn().mockReturnValue({ + createNamespacedCustomObject: jest.fn().mockResolvedValue({}), + patchNamespacedCustomObject: jest.fn().mockResolvedValue({}), + }), + }, +})) + +jest.mock('src/common/envalid', () => ({ + env: {}, +})) + +describe('sealed-secrets', () => { + describe('generateSealedSecretsKeyPair', () => { + it('should generate a valid key pair with certificate and private key', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'), + privateKeyToPem: jest + .fn() + .mockReturnValue('-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n'), + } as unknown as typeof pki, + } + + const result = generateSealedSecretsKeyPair(deps) + + expect(deps.pki.rsa.generateKeyPair).toHaveBeenCalledWith(4096) + expect(deps.pki.createCertificate).toHaveBeenCalled() + expect(mockCert.sign).toHaveBeenCalled() + expect(result.certificate).toContain('BEGIN CERTIFICATE') + expect(result.privateKey).toContain('BEGIN RSA PRIVATE KEY') + }) + + it('should set 10-year validity', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: {}, + privateKey: {}, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('cert'), + privateKeyToPem: jest.fn().mockReturnValue('key'), + } as unknown as typeof pki, + } + + generateSealedSecretsKeyPair(deps) + + const notBefore = mockCert.validity.notBefore.getFullYear() + const notAfter = mockCert.validity.notAfter.getFullYear() + expect(notAfter - notBefore).toBe(10) + }) + }) + + describe('getPemFromCertificate', () => { + it('should extract SPKI public key from a certificate', () => { + // Generate a real key pair and certificate for this test + const keys = pki.rsa.generateKeyPair(2048) + const cert = pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 1) + const attrs = [{ name: 'commonName', value: 'test' }] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.sign(keys.privateKey) + const certPem = pki.certificateToPem(cert) + + const result = getPemFromCertificate(certPem) + + expect(result).toContain('BEGIN PUBLIC KEY') + expect(result).toContain('END PUBLIC KEY') + }) + }) + + describe('createSealedSecretsKeySecret', () => { + it('should create secret if it does not exist', async () => { + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) + const deps = { + getK8sSecret: mockGetK8sSecret, + terminal, + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') + }) + + it('should skip creation if secret already exists', async () => { + const mockGetK8sSecret = jest.fn().mockResolvedValue({ 'tls.crt': 'existing-cert' }) + const deps = { + getK8sSecret: mockGetK8sSecret, + terminal, + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') + }) + }) + + describe('buildSecretToNamespaceMap', () => { + it('should group secrets by namespace and secret name with leaf key naming', async () => { + const secrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // All secrets now go to apl-secrets namespace + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('apl-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') + }) + + it('should skip kms.sops paths', async () => { + const secrets = { + kms: { sops: { provider: 'age', age: { publicKey: 'pk', privateKey: 'sk' } } }, + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('apl-secrets') + }) + + it('should skip users path (managed individually in apl-users namespace)', async () => { + const secrets = { + users: [ + { + email: 'admin@example.com', + firstName: 'Admin', + lastName: 'User', + initialPassword: 'pass', + groups: ['platform-admin'], + }, + ], + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result.find((m) => m.secretName === 'users-secrets')).toBeUndefined() + }) + + it('should handle teamConfig dynamic paths in apl-secrets namespace', async () => { + const secrets = { + teamConfig: { + 'team-alpha': { someSecret: 'value' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), + } + + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('apl-secrets') + expect(result[0].secretName).toBe('team-team-alpha-settings-secrets') + }) + + it('should filter out mappings with no data', async () => { + const secrets = {} + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(0) + }) + + it('should use leaf key naming for nested secret paths', async () => { + const secrets = { + apps: { + harbor: { core: { secret: 'core-secret-val' } }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.core.secret']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') + }) + + it('should put gitea secrets in apl-secrets namespace using convention naming', async () => { + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.gitea.adminPassword', + 'apps.gitea.postgresqlPassword', + 'apps.harbor.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // Harbor should use convention naming in apl-secrets ns + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('apl-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + + // Gitea should have a gitea-secrets mapping in apl-secrets ns + const giteaMapping = result.find((m) => m.secretName === 'gitea-secrets') + expect(giteaMapping).toBeDefined() + expect(giteaMapping!.namespace).toBe('apl-secrets') + expect(giteaMapping!.data).toHaveProperty('adminPassword', 'gitea-pass') + expect(giteaMapping!.data).toHaveProperty('postgresqlPassword', 'pg-pass') + }) + }) + + describe('createSealedSecretManifest', () => { + it('should produce correct SealedSecret structure', async () => { + const mapping = { + namespace: 'apl-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'my-password', secretKey: 'my-secret' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), + } + + const result = await createSealedSecretManifest('mock-pem', mapping, deps) + + expect(result.apiVersion).toBe('bitnami.com/v1alpha1') + expect(result.kind).toBe('SealedSecret') + expect(result.metadata.name).toBe('harbor-secrets') + expect(result.metadata.namespace).toBe('apl-secrets') + expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') + expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') + expect(result.spec.template.type).toBe('kubernetes.io/opaque') + expect(result.spec.template.metadata.name).toBe('harbor-secrets') + expect(result.spec.template.metadata.namespace).toBe('apl-secrets') + }) + + it('should call encryptSecretItem for each data key', async () => { + const mapping = { + namespace: 'apl-secrets', + secretName: 'gitea-secrets', + data: { key1: 'val1', key2: 'val2', key3: 'val3' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await createSealedSecretManifest('pem', mapping, deps) + + expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val3') + }) + }) + + describe('writeSealedSecretManifests', () => { + it('should write each manifest to the correct directory', async () => { + const manifests = [ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'apl-secrets', + }, + spec: { + encryptedData: { key: 'enc' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, + type: 'Opaque', + }, + }, + }, + ] + const deps = { + mkdir: jest.fn(), + writeFile: jest.fn(), + objectToYaml: jest.fn().mockReturnValue('yaml-content'), + terminal, + } + + await writeSealedSecretManifests(manifests, '/test', deps) + + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/namespaces/apl-secrets/sealedsecrets', { + recursive: true, + }) + expect(deps.writeFile).toHaveBeenCalledWith( + '/test/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml', + 'yaml-content', + ) + }) + }) + + describe('bootstrapSealedSecrets', () => { + it('should generate new key pair when no existing cert found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'apl-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + const mockManifest = { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'apl-secrets', + }, + spec: { + encryptedData: { adminPassword: 'encrypted' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, + type: 'Opaque', + }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), // No existing cert + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert-pem', + privateKey: 'key-pem', + }), + getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), + writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() + expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') + }) + + it('should use existing cert when found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'apl-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue('existing-cert-pem'), // Existing cert found + generateSealedSecretsKeyPair: jest.fn(), + getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue({}), + writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).not.toHaveBeenCalled() // Should NOT generate new key + expect(deps.createSealedSecretsKeySecret).not.toHaveBeenCalled() // Should NOT create secret + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should extract team names from secrets', async () => { + const secrets = { + teamConfig: { + alpha: { secret: 'val' }, + beta: { secret: 'val' }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert', + privateKey: 'key', + }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + createSealedSecretManifest: jest.fn(), + writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) + }) + }) + + describe('secret name derivation', () => { + it('should derive correct secret names via buildSecretToNamespaceMap', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + otomi: { adminPassword: 'pass' }, + obj: { provider: { linode: { secretAccessKey: 'key' } } }, + dns: { provider: { linode: { apiToken: 'token' } } }, + } + const result = await buildSecretToNamespaceMap(secrets, [], undefined, { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.harbor.adminPassword', + 'otomi.adminPassword', + 'obj.provider.linode.secretAccessKey', + 'dns.provider.linode.apiToken', + ]), + }) + const names = result.map((m) => m.secretName).sort() + expect(names).toEqual(['dns-secrets', 'harbor-secrets', 'obj-secrets', 'otomi-secrets']) + }) + }) + + describe('stripAllSecrets', () => { + it('should remove all secret paths from values', () => { + const values = { + apps: { + gitea: { adminPassword: 'secret', postgresqlPassword: 'pg-secret', resources: { cpu: '100m' } }, + }, + oidc: { clientID: 'otomi', clientSecret: 'my-secret', issuer: 'https://example.com' }, + } + const secretPaths = ['apps.gitea.adminPassword', 'apps.gitea.postgresqlPassword', 'oidc.clientSecret'] + + const result = stripAllSecrets(values, secretPaths) + + // Secret values should be removed + expect(result.apps.gitea.adminPassword).toBeUndefined() + expect(result.apps.gitea.postgresqlPassword).toBeUndefined() + expect(result.oidc.clientSecret).toBeUndefined() + // Non-secret values should be preserved + expect(result.apps.gitea.resources).toEqual({ cpu: '100m' }) + expect(result.oidc.clientID).toBe('otomi') + expect(result.oidc.issuer).toBe('https://example.com') + }) + + it('should not modify the original values object', () => { + const values = { + apps: { gitea: { adminPassword: 'secret' } }, + } + const secretPaths = ['apps.gitea.adminPassword'] + + stripAllSecrets(values, secretPaths) + + expect(values.apps.gitea.adminPassword).toBe('secret') + }) + }) + + describe('applySealedSecretManifests', () => { + const makeMockManifest = (name: string, namespace: string): SealedSecretManifest => ({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name, + namespace, + }, + spec: { + encryptedData: { key: 'encrypted-value' }, + template: { + immutable: false, + metadata: { name, namespace }, + type: 'kubernetes.io/opaque', + }, + }, + }) + + it('should apply manifests successfully', async () => { + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await applySealedSecretManifests(manifests, { terminal }) + + const { k8s: mockK8s } = require('src/common/k8s') + expect(mockK8s.custom().patchNamespacedCustomObject).toHaveBeenCalled() + }) + + it('should throw when manifests fail to apply', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + mockK8s.custom().patchNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) + + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await expect(applySealedSecretManifests(manifests, { terminal })).rejects.toThrow('apply failed') + }) + }) + + describe('restartSealedSecretsController', () => { + it('should succeed when deployment rolls out quickly', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + // Mock deployment that is already ready + mockK8s.app().readNamespacedDeployment.mockResolvedValue({ + spec: { replicas: 1 }, + status: { updatedReplicas: 1, availableReplicas: 1 }, + }) + + await restartSealedSecretsController({ terminal }) + + expect(mockK8s.app().patchNamespacedDeployment).toHaveBeenCalled() + }) + }) + + describe('createUserSealedSecretManifests', () => { + it('should create individual SealedSecret for each user', async () => { + const users = [ + { name: 'user1', email: 'user1@test.com', firstName: 'User', lastName: 'One', initialPassword: 'pass1' }, + { name: 'user2', email: 'user2@test.com', firstName: 'User', lastName: 'Two', initialPassword: 'pass2' }, + ] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(2) + expect(manifests[0].metadata.name).toBe('user1') + expect(manifests[0].metadata.namespace).toBe('apl-users') + expect(manifests[1].metadata.name).toBe('user2') + }) + + it('should skip users without id/name', async () => { + const users = [{ email: 'noname@test.com' }] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(0) + }) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts new file mode 100644 index 0000000000..d2618624f2 --- /dev/null +++ b/src/common/sealed-secrets.ts @@ -0,0 +1,657 @@ +import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' +import { encryptSecretItem } from '@linode/kubeseal-encrypt' +import { X509Certificate } from 'crypto' +import { existsSync } from 'fs' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' +import { cloneDeep, get, unset } from 'lodash' +import { pki } from 'node-forge' +import { join } from 'path' +import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' +import { terminal } from 'src/common/debug' +import { env } from 'src/common/envalid' +import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' +import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' +import { objectToYaml } from 'src/common/values' +import { parse as parseYaml } from 'yaml' + +const cmdName = 'sealed-secrets' +const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' + +/** + * Strip ALL x-secret fields from values before writing to disk. + * Secrets are stored exclusively in SealedSecrets and delivered to apps via ExternalSecrets. + * The values repo contains zero secret values. + */ +export function stripAllSecrets(values: Record, secretPaths: string[]): Record { + const stripped = cloneDeep(values) + for (const secretPath of secretPaths) { + unset(stripped, secretPath) + } + return stripped +} + +export interface SecretMapping { + namespace: string + secretName: string + data: Record +} + +export interface SealedSecretManifest { + apiVersion: string + kind: string + metadata: { + annotations: Record + name: string + namespace: string + } + spec: { + encryptedData: Record + template: { + immutable: boolean + metadata: { name: string; namespace: string } + type: string + } + } +} + +export interface SealedSecretsKeyPair { + certificate: string + privateKey: string +} + +export interface AppliedSecret { + namespace: string + secretName: string +} + +/** + * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. + * Follows the pattern from createCustomCA() in bootstrap.ts. + */ +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): SealedSecretsKeyPair => { + const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) + d.info('Generating sealed-secrets RSA key pair') + + const keys = deps.pki.rsa.generateKeyPair(4096) + const cert = deps.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 10) + + const attrs = [ + { name: 'countryName', value: 'NL' }, + { shortName: 'ST', value: 'Utrecht' }, + { name: 'localityName', value: 'Utrecht' }, + { name: 'organizationName', value: 'APL' }, + { shortName: 'OU', value: 'SealedSecrets' }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { + name: 'basicConstraints', + cA: true, + }, + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + keyEncipherment: true, + }, + ]) + cert.sign(keys.privateKey) + + const certificate = deps.pki.certificateToPem(cert).replaceAll('\r\n', '\n') + const privateKey = deps.pki.privateKeyToPem(keys.privateKey).replaceAll('\r\n', '\n') + + d.info('Generated sealed-secrets key pair') + return { certificate, privateKey } +} + +/** + * Extract SPKI PEM public key from a PEM-encoded X.509 certificate. + * Uses Node.js crypto.X509Certificate (same approach as getSealedSecretsPEM() in k8s.ts). + */ +export const getPemFromCertificate = (certificate: string): string => { + const x509 = new X509Certificate(certificate) + const exported = x509.publicKey.export({ format: 'pem', type: 'spki' }) + return typeof exported === 'string' ? exported : exported.toString('utf-8') +} + +/** + * Get the existing sealed-secrets certificate from the cluster if it exists. + * Returns the certificate PEM string or undefined if not found. + * Note: Uses k8s client directly instead of getK8sSecret() because PEM certificates + * are corrupted by the YAML parse step in getK8sSecret(). + */ +export const getExistingSealedSecretsCert = async (deps = { k8s, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) + + try { + const secret = await deps.k8s.core().readNamespacedSecret({ + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + }) + if (!secret?.data?.['tls.crt']) { + d.info('No existing sealed-secrets-key found') + return undefined + } + + d.info('Found existing sealed-secrets-key certificate') + return Buffer.from(secret.data['tls.crt'], 'base64').toString('utf-8') + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info('No existing sealed-secrets-key found') + return undefined + } + // When the cluster is unreachable (e.g., CI environment without a real cluster), + // treat it as no existing cert found and let bootstrap generate a new key pair. + d.info(`Could not reach cluster to check for existing cert: ${error instanceof Error ? error.message : error}`) + return undefined + } +} + +/** + * Create the sealed-secrets namespace and TLS secret in Kubernetes. + * The controller will pick up this pre-created key on startup. + * IMPORTANT: This only creates the secret if it doesn't already exist. + */ +export const createSealedSecretsKeySecret = async ( + certificate: string, + privateKey: string, + deps = { getK8sSecret, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) + + try { + await ensureNamespaceExists('sealed-secrets') + + // Check if secret already exists + const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') + if (existing) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } + + d.info('Creating sealed-secrets TLS secret') + + await k8s.core().createNamespacedSecret({ + namespace: 'sealed-secrets', + body: { + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { + 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + }, + }, + type: 'kubernetes.io/tls', + data: { + 'tls.crt': b64enc(certificate), + 'tls.key': b64enc(privateKey), + }, + }, + }) + + d.info('Created sealed-secrets TLS secret with key label') + } catch (error) { + // When the cluster is unreachable (e.g., CI/bootstrap without a real cluster), + // skip secret creation. The secret will be created during install when the cluster is available. + d.info( + `Could not create sealed-secrets-key in cluster (will be created during install): ${error instanceof Error ? error.message : error}`, + ) + } +} + +/** + * Find the group prefix for a secret path. + * Groups: teamConfig.X, apps.X, or a single top-level key (e.g., 'otomi', 'dns'). + */ +const findGroupPrefix = (secretPath: string): string | undefined => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) return `teamConfig.${teamMatch[1]}` + + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `apps.${appsMatch[1]}` + + // Top-level paths: use the first segment as the group prefix + const [firstSegment] = secretPath.split('.') + // Skip paths like 'kms' and 'users' which are handled separately + if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment + + return undefined +} + +/** + * Derive a K8s secret name from a secret path. + * Convention: all secrets follow {name}-secrets pattern. + * - teamConfig.X -> team-X-settings-secrets + * - apps.X -> X-secrets + * - topLevel -> topLevel-secrets + */ +const deriveSecretName = (secretPath: string): string => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) return `team-${teamMatch[1]}-settings-secrets` + + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `${appsMatch[1]}-secrets` + + return `${secretPath.split('.')[0]}-secrets` +} + +/** + * Build a mapping from secrets to their target namespaces and K8s secret names. + * Groups secret paths by namespace and secret name. + */ +export const buildSecretToNamespaceMap = async ( + secrets: Record, + teams: string[], + _allValues?: Record, + deps = { getSchemaSecretsPaths }, +): Promise => { + const secretPaths = await deps.getSchemaSecretsPaths(teams) + const flat = flattenObject(secrets) + + // Group by namespace + secretName + const groupMap = new Map() + + for (const secretPath of secretPaths) { + // Skip users path — user secrets are managed individually in apl-users namespace + if (secretPath === 'users') continue + + if (!findGroupPrefix(secretPath)) continue + + const secretName = deriveSecretName(secretPath) + const groupKey = `${SEALED_SECRETS_NAMESPACE}/${secretName}` + + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace: SEALED_SECRETS_NAMESPACE, secretName, data: {} }) + } + + const mapping = groupMap.get(groupKey)! + + // Find the group prefix (e.g., 'apps.harbor' for 'apps.harbor.adminPassword') + const groupPrefix = findGroupPrefix(secretPath) + + // Find all flat keys that match this secret path + for (const [flatKey, value] of Object.entries(flat)) { + if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { + // Use leaf key: strip the group prefix to get relative path + const relativePath = + groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) + ? flatKey.slice(groupPrefix.length + 1) + : flatKey + // Skip empty relative paths (happens when flatKey === groupPrefix) + if (!relativePath) continue + const dataKey = relativePath.replace(/\./g, '_') + if (value !== undefined && value !== null && value !== '') { + mapping.data[dataKey] = String(value) + } + } + } + } + + // Filter out empty mappings + return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) +} + +/** + * Create a SealedSecret manifest by encrypting each data value. + * Follows the pattern from createCatalogSealedSecret() in migrate.ts. + */ +export const createSealedSecretManifest = async ( + pem: string, + mapping: SecretMapping, + deps = { encryptSecretItem }, +): Promise => { + const encryptedData: Record = {} + for (const [key, value] of Object.entries(mapping.data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, mapping.namespace, value) + } + + return { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: mapping.secretName, + namespace: mapping.namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: mapping.secretName, namespace: mapping.namespace }, + type: 'kubernetes.io/opaque', + }, + }, + } +} + +/** + * Write SealedSecret manifests to the env/manifests/namespaces directory. + */ +export const writeSealedSecretManifests = async ( + manifests: SealedSecretManifest[], + envDir: string, + deps = { mkdir, writeFile, objectToYaml, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) + + for (const manifest of manifests) { + const dir = `${envDir}/${SEALED_SECRETS_MANIFESTS_SUBDIR}/${manifest.metadata.namespace}/sealedsecrets` + await deps.mkdir(dir, { recursive: true }) + const filePath = `${dir}/${manifest.metadata.name}.yaml` + d.info(`Writing sealed secret to ${filePath}`) + await deps.writeFile(filePath, deps.objectToYaml(manifest)) + } +} + +/** + * Apply a single SealedSecret manifest using server-side apply (create-or-update). + * Uses the same pattern as applyArgocdApp() in apply-as-apps.ts. + */ +const applySealedSecretResource = async (manifest: SealedSecretManifest): Promise => { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace: manifest.metadata.namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + fieldManager: 'apl-operator', + force: true, + }, + setHeaderOptions('Content-Type', PatchStrategy.ServerSideApply), + ) +} + +/** + * Apply SealedSecret manifests to the Kubernetes cluster. + * Creates namespaces if needed and applies the SealedSecret resources. + */ +export const applySealedSecretManifests = async ( + manifests: SealedSecretManifest[], + deps = { terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) + + // Group manifests by namespace + const byNamespace = new Map() + for (const manifest of manifests) { + const ns = manifest.metadata.namespace + if (!byNamespace.has(ns)) { + byNamespace.set(ns, []) + } + byNamespace.get(ns)!.push(manifest) + } + + // Ensure namespaces exist and apply manifests + for (const [namespace, nsManifests] of byNamespace) { + await ensureNamespaceExists(namespace) + + for (const manifest of nsManifests) { + d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) + await applySealedSecretResource(manifest) + } + } + + d.info(`Applied ${manifests.length} SealedSecret manifests to cluster`) +} + +/** + * Read and apply all SealedSecret manifests from the env/manifests/namespaces directory. + * This should be called during install, after the sealed-secrets controller is deployed. + * Returns the list of applied secrets (namespace + secretName) so callers can wait for them. + */ +export const applySealedSecretManifestsFromDir = async ( + envDir: string, + deps = { terminal, readdir, readFile, existsSync }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) + const manifestsDir = join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR) + + if (!deps.existsSync(manifestsDir)) { + d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) + return [] + } + + d.info(`Applying SealedSecret manifests from ${manifestsDir}`) + + // Read all namespace directories + const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) + const appliedSecrets: AppliedSecret[] = [] + + for (const nsEntry of namespaces) { + if (!nsEntry.isDirectory()) continue + const namespace = nsEntry.name + const sealedSecretsDir = join(manifestsDir, namespace, 'sealedsecrets') + + if (!deps.existsSync(sealedSecretsDir)) continue + + await ensureNamespaceExists(namespace) + + // Read all YAML files in the sealedsecrets subdirectory + const files = await deps.readdir(sealedSecretsDir) + for (const file of files) { + if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue + const filePath = join(sealedSecretsDir, file) + d.info(`Applying SealedSecret from ${filePath}`) + + try { + const content = await deps.readFile(filePath, 'utf-8') + const manifest = parseYaml(content) as SealedSecretManifest + + await applySealedSecretResource(manifest) + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) + } catch (error) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) + } + } + } + + d.info(`Applied ${appliedSecrets.length} SealedSecret manifests from directory`) + return appliedSecrets +} + +/** + * Restart the sealed-secrets controller to ensure it uses the correct key. + * This is needed because if the controller starts before the sealed-secrets-key secret exists, + * it will generate its own key. Restarting forces it to pick up the existing key. + */ +export const restartSealedSecretsController = async (deps = { terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) + d.info('Restarting sealed-secrets controller to ensure correct key is used') + + try { + await k8s.app().patchNamespacedDeployment( + { + name: 'sealed-secrets', + namespace: 'sealed-secrets', + body: { + spec: { + template: { + metadata: { + annotations: { + 'kubectl.kubernetes.io/restartedAt': new Date().toISOString(), + }, + }, + }, + }, + }, + }, + setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), + ) + } catch (error) { + d.warn(`Failed to restart sealed-secrets controller: ${error}`) + return + } + + d.info('Waiting for sealed-secrets controller rollout') + const start = Date.now() + while (Date.now() - start < env.SEALED_SECRETS_TIMEOUT_MS) { + try { + const deployment = await k8s.app().readNamespacedDeployment({ + name: 'sealed-secrets', + namespace: 'sealed-secrets', + }) + const desired = deployment.spec?.replicas ?? 1 + const updated = deployment.status?.updatedReplicas ?? 0 + const available = deployment.status?.availableReplicas ?? 0 + if (updated >= desired && available >= desired) { + d.info('Sealed-secrets controller restarted successfully') + return + } + } catch { + // Ignore transient read errors during rollout + } + await new Promise((resolve) => setTimeout(resolve, env.SEALED_SECRETS_INTERVAL_MS)) + } + d.warn('Rollout status check timed out') +} + +/** + * Create individual SealedSecret manifests for each user in the apl-users namespace. + * Each user gets their own SealedSecret with all fields encrypted. + */ +export const createUserSealedSecretManifests = async ( + users: any[], + pem: string, + deps = { encryptSecretItem, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createUserSealedSecretManifests`) + const namespace = 'apl-users' + const manifests: SealedSecretManifest[] = [] + + for (const user of users) { + const userId = user.name || user.id + if (!userId) { + d.warn('Skipping user without id/name') + continue + } + + const data: Record = { + email: user.email || '', + firstName: user.firstName || '', + lastName: user.lastName || '', + initialPassword: user.initialPassword || '', + isPlatformAdmin: String(user.isPlatformAdmin ?? false), + isTeamAdmin: String(user.isTeamAdmin ?? false), + teams: JSON.stringify(user.teams || []), + } + + const encryptedData: Record = {} + for (const [key, value] of Object.entries(data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, namespace, value) + } + + manifests.push({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: userId, + namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: userId, namespace }, + type: 'kubernetes.io/opaque', + }, + }, + }) + } + + d.info(`Created ${manifests.length} individual user SealedSecret manifests`) + return manifests +} + +/** + * Get the PEM public key from the existing sealed-secrets certificate in the cluster, + * or generate a new RSA key pair, store it in the cluster, and return its PEM. + */ +export const getOrCreateSealedSecretsPem = async ( + deps = { + terminal, + getExistingSealedSecretsCert, + getPemFromCertificate, + generateSealedSecretsKeyPair, + createSealedSecretsKeySecret, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:getOrCreateSealedSecretsPem`) + const existingCert = await deps.getExistingSealedSecretsCert() + if (existingCert) { + d.info('Using existing sealed-secrets certificate') + return deps.getPemFromCertificate(existingCert) + } + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + return deps.getPemFromCertificate(certificate) +} + +export const bootstrapSealedSecrets = async ( + secrets: Record, + envDir: string, + allValues?: Record, + deps = { + terminal, + generateSealedSecretsKeyPair, + getPemFromCertificate, + createSealedSecretsKeySecret, + getExistingSealedSecretsCert, + buildSecretToNamespaceMap, + createSealedSecretManifest, + writeSealedSecretManifests, + createUserSealedSecretManifests, + encryptSecretItem, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) + d.info('Bootstrapping sealed secrets') + + // 1. Get or create the sealed-secrets PEM public key + const pem = await getOrCreateSealedSecretsPem({ + terminal: deps.terminal, + getExistingSealedSecretsCert: deps.getExistingSealedSecretsCert, + getPemFromCertificate: deps.getPemFromCertificate, + generateSealedSecretsKeyPair: deps.generateSealedSecretsKeyPair, + createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, + }) + + // 5. Build secret-to-namespace mapping + const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) + + // 6. Create SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + + // 7. Create individual user SealedSecrets in apl-users namespace + const { users } = secrets + if (Array.isArray(users) && users.length > 0) { + const userManifests = await deps.createUserSealedSecretManifests(users, pem, { + encryptSecretItem: deps.encryptSecretItem, + terminal: deps.terminal, + }) + manifests.push(...userManifests) + } + + // 8. Write SealedSecret manifests to disk + // Note: These manifests are applied later during install, after the sealed-secrets + // controller is deployed and the SealedSecret CRD is available. + await deps.writeSealedSecretManifests(manifests, envDir) + + d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) +} diff --git a/src/common/values.test.ts b/src/common/values.test.ts index 06f76fa4e6..3af2dead81 100644 --- a/src/common/values.test.ts +++ b/src/common/values.test.ts @@ -60,6 +60,7 @@ describe('generateSecrets', () => { deps = { terminal, getValuesSchema: jest.fn().mockReturnValue(schema), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), } }) it('should generate new secrets and return only secrets', async () => { @@ -72,4 +73,26 @@ describe('generateSecrets', () => { const res = await generateSecrets(valuesWithExisting, deps) expect(res.nested.twoStage).toBe('exists') }) + it('should include team secrets with expanded paths', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.demo.settings.password', 'team-secret-pw') + + deps.getSchemaSecretsPaths.mockResolvedValue(['teamConfig.demo.settings.password']) + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledWith(['demo']) + expect(res.teamConfig.demo.settings.password).toBe('team-secret-pw') + }) + it('should not call getSchemaSecretsPaths when no dynamic teams exist', async () => { + const res = await generateSecrets(values, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + expect(res).toEqual(expected) + }) + it('should exclude admin team from dynamic team expansion', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.admin.settings.password', 'admin-pw') + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + }) }) diff --git a/src/common/values.ts b/src/common/values.ts index 157ee2fc2c..e8270d034a 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, omit, pick, set } from 'lodash' +import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -9,6 +9,7 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' +import { saveValues } from './repo' import { extract, flattenObject, @@ -19,8 +20,6 @@ import { pkg, removeBlankAttributes, } from './utils' - -import { saveValues } from './repo' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -114,22 +113,13 @@ export const writeValuesToFile = async ( /** * Writes new values to the repo. Will keep the original values if `overwrite` is `false`. + * Secret values are written as-is — they are protected by SealedSecrets on the cluster side, + * and child secrets are derived via ESO ExternalSecret CRs. */ export const writeValues = async (inValues: Record, overwrite = false): Promise => { const d = terminal('common:values:writeValues') d.debug('Writing values: ', inValues) - hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = inValues - const teams = Object.keys(get(inValues, 'teamConfig', {})) - const cleanSecretPaths = await getSchemaSecretsPaths(teams) - d.debug('cleanSecretPaths: ', cleanSecretPaths) - // separate out the secrets - const secrets = removeBlankAttributes(pick(values, cleanSecretPaths)) - d.debug('secrets: ', JSON.stringify(secrets, null, 2)) - // from the plain values - const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, plainValues, secrets) - + await saveValues(env.ENV_DIR, inValues, {}) d.info('All values were written to ENV_DIR') } @@ -154,6 +144,7 @@ export const generateSecrets = async ( deps = { terminal, getValuesSchema, + getSchemaSecretsPaths, }, ): Promise> => { const d = deps.terminal('common:values:generateSecrets') @@ -177,6 +168,16 @@ export const generateSecrets = async ( // Only return values that have x-secrets prop and are now fully templated: const templatePaths = Object.keys(flattenObject(schemaSecrets)) const res = pick(allSecrets, templatePaths) + + // Template paths use schema patternProperties regex keys which don't match concrete team names. + // Expand team paths so team secrets are included in the result. + const teamNames = Object.keys(get(values, 'teamConfig', {})).filter((t) => t !== 'admin') + if (teamNames.length > 0) { + const expandedPaths = await deps.getSchemaSecretsPaths(teamNames) + const teamSecrets = pick(allSecrets, expandedPaths) + merge(res, teamSecrets) + } + d.debug('generateSecrets result: ', res) return res } diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index d37318546c..b3c87cec13 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -100,6 +100,7 @@ export class AplOperator { } else { await this.aplOps.apply() } + this.d.info(`[${trigger}] Apply process completed`) await updateApplyState({ diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index f122d58180..586ecbbd0d 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -1,9 +1,20 @@ +import { ApiException } from '@kubernetes/client-node' import * as gitConfig from '../common/git-config' -import * as hf from '../common/hf' import * as k8s from '../common/k8s' +import * as utils from '../common/utils' import { AplOperations } from './apl-operations' import { Installer } from './installer' +const mockZx = jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }), + }), +}) + +jest.mock('zx', () => ({ + $: (...args: any[]) => mockZx(...args), +})) + jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -14,27 +25,32 @@ jest.mock('../common/debug', () => ({ })) jest.mock('../common/k8s', () => ({ - deletePendingHelmReleases: jest.fn(), getK8sConfigMap: jest.fn(), getK8sSecret: jest.fn(), createUpdateConfigMap: jest.fn(), createUpdateGenericSecret: jest.fn(), + deletePendingHelmReleases: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), k8s: { core: jest.fn(), }, })) -jest.mock('../common/hf', () => ({ - hfValues: jest.fn(), +jest.mock('../common/utils', () => ({ + ...jest.requireActual('../common/utils'), + loadYaml: jest.fn(), +})) + +jest.mock('../common/envalid', () => ({ + env: { VALUES_INPUT: '/tmp/test-values.yaml' }, })) jest.mock('../common/git-config', () => ({ - getGitCredentials: jest.fn().mockResolvedValue(undefined), - getGitConfigData: jest.fn().mockResolvedValue(undefined), - getStoredGitRepoConfig: jest.fn().mockResolvedValue(undefined), - setGitConfig: jest.fn().mockResolvedValue(undefined), - GIT_CONFIG_SECRET_NAME: 'apl-git-credentials', - GIT_CONFIG_NAMESPACE: 'apl-operator', + getStoredGitRepoConfig: jest.fn(), +})) + +jest.mock('src/common/bootstrap', () => ({ + recoverFromGit: jest.fn().mockResolvedValue(undefined), })) jest.mock('src/cmd/traces', () => ({ @@ -54,10 +70,9 @@ describe('Installer', () => { jest.clearAllMocks() jest.useFakeTimers() - // Save original environment variables - process.env.SOPS_AGE_KEY = '' - - mockCoreApi = {} + mockCoreApi = { + createNamespacedSecret: jest.fn().mockResolvedValue(undefined), + } ;(k8s.k8s.core as jest.Mock).mockReturnValue(mockCoreApi) mockAplOps = { @@ -244,10 +259,25 @@ describe('Installer', () => { }) describe('getInstallationState', () => { - test('should return isInstalled=true and standard mode when status is completed', async () => { + test('should return isInstalled=true and standard mode when status is completed and git repo has main branch', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) + // getStoredGitRepoConfig returns valid config + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) + // git ls-remote succeeds (main branch exists) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const state = await installer.getInstallationState() @@ -256,9 +286,47 @@ describe('Installer', () => { expect(state.installationMode).toBe('standard') }) + test('should return false when status is completed but git repo has no main branch', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) + // git ls-remote fails (main branch does not exist) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 2 }), + }), + }) + + const state = await installer.getInstallationState() + + expect(state.isInstalled).toBe(false) + }) + test('should return isInstalled=true when ConfigMap does not exist (migrated state)', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue(null) ;(k8s.createUpdateConfigMap as jest.Mock).mockResolvedValue(undefined) + // Git verification will run since isInstalled=true for migrated state + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const state = await installer.getInstallationState() @@ -296,248 +364,236 @@ describe('Installer', () => { expect(state.installationMode).toBe('recovery') }) + + test('should return true when git verification fails (gitea not ready)', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + // getStoredGitRepoConfig throws (cluster issues) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('connection refused')) + + const state = await installer.getInstallationState() + + // Should assume installed when verification can't be performed + expect(state.isInstalled).toBe(true) + }) }) describe('setEnvAndCreateSecrets', () => { - test('should use existing credentials from apl-git-credentials secret when available', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) // apl-sops-secrets + test('should set SOPS_AGE_KEY when secret exists', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) - const result = await installer.setEnvAndCreateSecrets() + await installer.setEnvAndCreateSecrets() expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') - expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') + expect(process.env.SOPS_AGE_KEY).toBe('AGE-SECRET-KEY-1ABC') }) - test('should handle failure when SOPS key not found in secret', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(null) + test('should not throw when secret does not exist', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - await expect(installer.setEnvAndCreateSecrets()).rejects.toThrow('SOPS_AGE_KEY not found in secret') + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) - }) - describe('ensureRecoveryPrerequisites', () => { - test('should pass when stored git config and sops secret are present', async () => { - ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - authenticatedUrl: 'https://admin:s3cret@github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - username: 'admin', - password: 's3cret', - }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) + test('should not throw when getK8sSecret fails', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('Not found')) - await expect(installer.ensureRecoveryPrerequisites()).resolves.toBeUndefined() - expect(gitConfig.getStoredGitRepoConfig).toHaveBeenCalledTimes(1) - expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) - test('should fail when sops secret is missing', async () => { - ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - authenticatedUrl: 'https://admin:s3cret@github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - username: 'admin', - password: 's3cret', - }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) + test('should handle secret without SOPS_AGE_KEY field', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ OTHER_KEY: 'value' }) - await expect(installer.ensureRecoveryPrerequisites()).rejects.toThrow( - 'KMS/SOPS config not found in apl-sops-secrets secret', - ) + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) }) - describe('ensureSecretsAndConfig', () => { - const mockValues = { - otomi: { - git: { - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - username: 'admin', - password: 's3cret', - }, - }, - kms: { - sops: { - age: { - privateKey: 'AGE-SECRET-KEY-1234', + describe('applyRecoveryManifests', () => { + test('should create secrets from manifest items', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + apiVersion: 'v1', + kind: 'List', + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key2', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydDI=', 'tls.key': 'a2V5Mg==' }, + }, + ], + }, }, }, - }, - } - - test('should skip when hfValues returns undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() + }) - expect(gitConfig.getGitCredentials).not.toHaveBeenCalled() - expect(k8s.getK8sSecret).not.toHaveBeenCalled() - expect(gitConfig.getGitConfigData).not.toHaveBeenCalled() + await installer.applyRecoveryManifests() + + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('sealed-secrets') + expect(k8s.ensureNamespaceExists).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith({ + namespace: 'sealed-secrets', + body: { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + }) }) - test('should not recreate resources when all exist', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', + test('should handle 409 conflict (secret already exists)', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==' }, + }, + ], + }, + }, + }, }) - await installer.ensureSecretsAndConfig() + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(409, 'Conflict', {}, {})) - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() - expect(gitConfig.setGitConfig).not.toHaveBeenCalled() + await expect(installer.applyRecoveryManifests()).resolves.not.toThrow() }) - test('should recreate git credentials when missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', + test('should skip when no manifests present', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { mode: 'recovery' }, }) - await installer.ensureSecretsAndConfig() + await installer.applyRecoveryManifests() - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() }) - test('should recreate sops secret when missing and age key exists in values', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', + test('should skip when items array is empty', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { recovery: { manifests: { items: [] } } }, }) - await installer.ensureSecretsAndConfig() + await installer.applyRecoveryManifests() - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() }) - test('should not recreate sops secret when missing but no age key in values', async () => { - const valuesWithoutSops = { - ...mockValues, - kms: { sops: { age: {} } }, - } - ;(hf.hfValues as jest.Mock).mockResolvedValue(valuesWithoutSops) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', + test('should rethrow non-409 errors', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + data: {}, + }, + ], + }, + }, + }, }) - await installer.ensureSecretsAndConfig() + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(500, 'Internal Server Error', {}, {})) - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() + await expect(installer.applyRecoveryManifests()).rejects.toThrow() }) - test('should recreate git config when repoUrl is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - branch: 'main', - email: 'pipeline@cluster.local', + test('should use default namespace when metadata.namespace is not set', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'my-secret' }, + data: { key: 'val' }, + }, + ], + }, + }, + }, }) - await installer.ensureSecretsAndConfig() + await installer.applyRecoveryManifests() - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('default') + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith( + expect.objectContaining({ + namespace: 'default', + }), + ) }) + }) - test('should recreate git config when branch is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - email: 'pipeline@cluster.local', + describe('ensureRecoveryPrerequisites', () => { + test('should succeed without SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() }) - test('should recreate git config when email is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', + test('should succeed with SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() }) - test('should recreate git config when configData is undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', + test('should succeed with empty SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', }) - }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({}) - test('should recreate all resources when all are missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) - await installer.ensureSecretsAndConfig() + test('should throw when git config is missing', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('Git config not found')) - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) + await expect(installer.ensureRecoveryPrerequisites()).rejects.toThrow('Git config not found') }) }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 81a9846e2e..98e1dd9849 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,18 +1,21 @@ +import { ApiException } from '@kubernetes/client-node' import * as process from 'node:process' import { runTraceCollectionLoop } from 'src/cmd/traces' import { recoverFromGit } from 'src/common/bootstrap' import { APL_OPERATOR_NS, APL_OPERATOR_STATUS_CM } from 'src/common/constants' +import { $ } from 'zx' import { terminal } from '../common/debug' +import { env } from '../common/envalid' +import { getStoredGitRepoConfig, GIT_CONFIG_NAMESPACE } from '../common/git-config' import { - getGitConfigData, - getGitCredentials, - getStoredGitRepoConfig, - GIT_CONFIG_NAMESPACE, - GIT_CONFIG_SECRET_NAME, - setGitConfig, -} from '../common/git-config' -import { hfValues } from '../common/hf' -import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' + createUpdateConfigMap, + deletePendingHelmReleases, + ensureNamespaceExists, + getK8sConfigMap, + getK8sSecret, + k8s, +} from '../common/k8s' +import { loadYaml } from '../common/utils' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -45,9 +48,33 @@ export class Installer { isInstalled = status === 'completed' } + if (isInstalled) { + // Verify the git repo actually has content - the previous install may have + // marked status as completed but the pod was killed before the git push finished + const gitRepoHasContent = await this.verifyGitRepoHasMainBranch() + if (!gitRepoHasContent) { + this.d.warn('Installation marked as completed but git repo has no main branch - will re-install') + isInstalled = false + } + } + return { installationMode, isInstalled } } + private async verifyGitRepoHasMainBranch(): Promise { + try { + const gitConfig = await getStoredGitRepoConfig() + if (!gitConfig) return true // Can't verify without config, assume fine + const result = await $`git ls-remote --exit-code --heads ${gitConfig.authenticatedUrl} main`.nothrow().quiet() + return result.exitCode === 0 + } catch { + // If we can't check (e.g. gitea not ready yet), assume it's fine + // The operator will detect the issue later during git polling + this.d.warn('Could not verify git repo - may not be ready yet') + return true + } + } + public async recoverFromGit(): Promise { while (true) { try { @@ -80,9 +107,48 @@ export class Installer { public async ensureRecoveryPrerequisites(): Promise { await getStoredGitRepoConfig() + // SOPS is optional — sealed-secrets clusters don't have it const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret || Object.keys(sopsSecret).length === 0) { - throw new Error('KMS/SOPS config not found in apl-sops-secrets secret') + if (sopsSecret && Object.keys(sopsSecret).length > 0) { + this.d.info('SOPS configuration found for recovery') + } else { + this.d.info('No SOPS configuration — sealed-secrets mode recovery') + } + } + + public async applyRecoveryManifests(): Promise { + const values = (await loadYaml(env.VALUES_INPUT)) as Record + const items = values?.installation?.recovery?.manifests?.items + if (!Array.isArray(items) || items.length === 0) { + this.d.info('No recovery manifests to apply') + return + } + + this.d.info(`Applying ${items.length} recovery manifest(s)`) + for (const item of items) { + const namespace = item.metadata?.namespace || 'default' + const name = item.metadata?.name + await ensureNamespaceExists(namespace) + + try { + await k8s.core().createNamespacedSecret({ + namespace, + body: { + apiVersion: item.apiVersion, + kind: item.kind, + metadata: { name, namespace, labels: item.metadata?.labels }, + type: item.type, + data: item.data, + }, + }) + this.d.info(`Created recovery secret ${namespace}/${name}`) + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + this.d.info(`Recovery secret ${namespace}/${name} already exists, skipping`) + } else { + throw error + } + } } } @@ -105,14 +171,23 @@ export class Installer { // Run the installation sequence await this.updateInstallationStatus('in-progress', attemptNumber) await this.aplOps.install() - await this.ensureSecretsAndConfig() await this.updateInstallationStatus('completed', attemptNumber) return } catch (error) { + const errorMessage = getErrorMessage(error) + this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) await this.updateInstallationStatus('failed', attemptNumber) - this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`, getErrorMessage(error)) - // Wait 1 second before retrying + + // Clean up stuck Helm releases (e.g. pending-install, pending-upgrade) + // so the next retry can proceed without "another operation is in progress" errors + try { + await deletePendingHelmReleases() + } catch (cleanupError) { + this.d.warn('Failed to clean up pending Helm releases:', getErrorMessage(cleanupError)) + } + + this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) await new Promise((resolve) => setTimeout(resolve, 1000)) } } @@ -141,59 +216,21 @@ export class Installer { } public async setEnvAndCreateSecrets(): Promise { - this.d.debug('Retrieving or creating git credentials') + this.d.debug('Setting up environment') await this.setupSopsEnvironment() } - private async setupSopsEnvironment() { - const aplSopsSecret = await getK8sSecret('apl-sops-secrets', APL_OPERATOR_NS) - - if (!aplSopsSecret?.SOPS_AGE_KEY) { - throw new Error('SOPS_AGE_KEY not found in secret') - } - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY - } - - // public for testing. This method should only be used if you are certain there are values locally. - async ensureSecretsAndConfig(): Promise { - this.d.info('Verifying secrets and config after installation') - const values = (await hfValues()) as Record - if (!values) { - this.d.warn('Could not retrieve hfValues, skipping secrets/config verification') - return - } - - const otomiGit = values?.otomi?.git - const agePrivateKey = values?.kms?.sops?.age?.privateKey - - // Ensure apl-git-credentials secret - const credentials = await getGitCredentials() - if (!credentials) { - this.d.info('Recreating apl-git-credentials secret') - await createUpdateGenericSecret(k8s.core(), GIT_CONFIG_SECRET_NAME, GIT_CONFIG_NAMESPACE, { - username: otomiGit?.username, - password: otomiGit?.password, - }) - } - - // Ensure apl-sops-secrets secret - const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret?.SOPS_AGE_KEY && agePrivateKey) { - this.d.info('Recreating apl-sops-secrets secret') - await createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', GIT_CONFIG_NAMESPACE, { - SOPS_AGE_KEY: agePrivateKey, - }) - } - - // Ensure apl-git-config configmap - const configData = await getGitConfigData() - if (!configData?.repoUrl || !configData?.branch || !configData?.email) { - this.d.info('Recreating apl-git-config configmap') - await setGitConfig({ - repoUrl: otomiGit?.repoUrl, - branch: otomiGit?.branch, - email: otomiGit?.email, - }) + private async setupSopsEnvironment(): Promise { + try { + const aplSopsSecret = await getK8sSecret('apl-sops-secrets', APL_OPERATOR_NS) + if (!aplSopsSecret?.SOPS_AGE_KEY) { + this.d.info('SOPS_AGE_KEY not found — cluster may already use SealedSecrets') + return + } + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.info('SOPS environment configured') + } catch (error) { + this.d.info('Could not read apl-sops-secrets — cluster may already use SealedSecrets') } } } diff --git a/src/operator/main.ts b/src/operator/main.ts index 203da7390d..04ad2861b1 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -76,8 +76,9 @@ async function main(): Promise { if (isInstalled) { d.info('Installation already completed, skipping install steps') } else if (isRecoveryMode) { - d.info('Recovery mode enabled, checking external git and kms prerequisites') + d.info('Recovery mode enabled, checking prerequisites') await installer.ensureRecoveryPrerequisites() + await installer.applyRecoveryManifests() await installer.recoverFromGit() d.info('Recovery installation completed, switching installation mode to standard') await installer.resetRecoveryModeToStandard() @@ -88,8 +89,10 @@ async function main(): Promise { await installer.reconcileInstall() } + // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) + await installer.setEnvAndCreateSecrets() + // Phase 2: Set environment variables and start operator for GitOps operations - // await installer.setEnvAndCreateSecrets() const config = await loadConfig(aplOps) const operator = new AplOperator(config) handleTerminationSignals(operator) diff --git a/src/server.ts b/src/server.ts index d989d8fc1e..347cb12e5b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,13 +1,7 @@ -import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser' import express, { Request, Response } from 'express' -import { copyFile } from 'fs/promises' import { Server } from 'http' -import { bootstrapSops } from 'src/cmd/bootstrap' -import { decrypt, encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { hfValues } from './common/hf' -import { setValuesFile, unsetValuesFile } from './common/repo' -import { loadYaml, rootDir } from './common/utils' import { objectToYaml } from './common/values' const d = terminal('server') @@ -24,47 +18,8 @@ app.get('/', async (req: Request, res: Response): Promise => { type QueryParams = { envDir: string - files?: string[] } -app.get('/init', async (req: Request, res: Response): Promise => { - const { envDir } = req.query as QueryParams - try { - d.log('Request to initialize values repo on', envDir) - await decrypt(envDir) - res.status(200).send('ok') - } catch (error) { - d.error(error) - res.status(500).send(`${error}`) - } -}) - -app.get('/prepare', async (req: Request, res: Response): Promise => { - const { envDir, files } = req.query as QueryParams - try { - d.log('Request to prepare values repo on', envDir) - const file = '.editorconfig' - await copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - await bootstrapSops(envDir) - await setValuesFile(envDir) - // Encrypt ensures that a brand new secret file is encrypted in place - await encrypt(envDir, ...(files ?? [])) - // Decrypt ensures that a brand new encrypted secret file is decrypted to the .dec file - await decrypt(envDir, ...(files ?? [])) - res.status(200).send('ok') - } catch (error) { - const err = `${error}` - let status = 500 - d.error(`Request to prepare values went wrong: ${err}`) - if (err.includes('Values validation FAILED')) { - status = 422 - } - res.status(status).send(err) - } finally { - await unsetValuesFile(envDir) - } -}) - function parseBoolean(string: any, defaultValue = false): boolean { return string === 'true' ? true : string === 'false' ? false : defaultValue } @@ -87,13 +42,6 @@ app.get('/otomi/values', async (req: Request, res: Response): Promise => { } }) -app.get('/apl/schema', async (req: Request, res: Response): Promise => { - const schema = await loadYaml(`${rootDir}/values-schema.yaml`) - const derefSchema = await $RefParser.dereference(schema as JSONSchema) - res.setHeader('Content-type', 'application/json') - res.status(200).send(derefSchema) -}) - export const startServer = (): void => { server = app .listen(17771, '127.0.0.1', () => { diff --git a/tests/bootstrap/input-local-dev.yaml b/tests/bootstrap/input-local-dev.yaml index ccaaafd9eb..1f7cdfd0e5 100644 --- a/tests/bootstrap/input-local-dev.yaml +++ b/tests/bootstrap/input-local-dev.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: true diff --git a/tests/bootstrap/input.yaml b/tests/bootstrap/input.yaml index 0cb49d23af..64d426b912 100644 --- a/tests/bootstrap/input.yaml +++ b/tests/bootstrap/input.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: false diff --git a/tests/fixtures/env/apps/cert-manager.yaml b/tests/fixtures/env/apps/cert-manager.yaml index be2c7211a3..d484facf18 100644 --- a/tests/fixtures/env/apps/cert-manager.yaml +++ b/tests/fixtures/env/apps/cert-manager.yaml @@ -3,6 +3,28 @@ metadata: name: cert-manager labels: {} spec: + customRootCA: | + -----BEGIN CERTIFICATE----- + MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N + M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX + xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl + g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw + K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 + xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV + AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB + BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 + tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg + ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y + yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv + ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN + FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI + -----END CERTIFICATE----- externallyManagedTlsSecretName: mysecret issuer: externally-managed-tls-secret _rawValues: {} diff --git a/tests/fixtures/env/apps/secrets.argocd.yaml b/tests/fixtures/env/apps/secrets.argocd.yaml deleted file mode 100644 index aabb4c4109..0000000000 --- a/tests/fixtures/env/apps/secrets.argocd.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - redisPassword: somepassword -name: argocd-redis -metadata: - name: argocd diff --git a/tests/fixtures/env/apps/secrets.cert-manager.yaml b/tests/fixtures/env/apps/secrets.cert-manager.yaml deleted file mode 100644 index 671916d60e..0000000000 --- a/tests/fixtures/env/apps/secrets.cert-manager.yaml +++ /dev/null @@ -1,55 +0,0 @@ -kind: AplApp -spec: - customRootCA: | - -----BEGIN CERTIFICATE----- - MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N - M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX - xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl - g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw - K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 - xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV - AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB - BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 - tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg - ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y - yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv - ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN - FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI - -----END CERTIFICATE----- - customRootCAKey: | - -----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ - ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd - BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K - GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq - pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH - NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 - 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj - VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr - V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT - UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN - joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B - EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ - AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE - 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT - MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw - UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa - DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl - BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM - wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC - bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ - E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K - 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK - CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H - rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX - EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= - -----END RSA PRIVATE KEY----- -name: cert-manager -metadata: - name: cert-manager diff --git a/tests/fixtures/env/apps/secrets.gitea.yaml b/tests/fixtures/env/apps/secrets.gitea.yaml deleted file mode 100644 index 25e72d933c..0000000000 --- a/tests/fixtures/env/apps/secrets.gitea.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - postgresqlPassword: postgresqlPassword -name: gitea -metadata: - name: gitea diff --git a/tests/fixtures/env/apps/secrets.grafana.yaml b/tests/fixtures/env/apps/secrets.grafana.yaml deleted file mode 100644 index 984e5171de..0000000000 --- a/tests/fixtures/env/apps/secrets.grafana.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: grafana -metadata: - name: grafana diff --git a/tests/fixtures/env/apps/secrets.harbor.yaml b/tests/fixtures/env/apps/secrets.harbor.yaml deleted file mode 100644 index 84dcfe2783..0000000000 --- a/tests/fixtures/env/apps/secrets.harbor.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: AplApp -spec: - adminPassword: harborsomesecretvalue - core: - secret: vQFMm9Qk0pTUF3MK - xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi - jobservice: - secret: CfpanIkcGWz3wGLO - registry: - secret: PjHGEdmPhrmNrEkj - credentials: - htpasswd: admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6 - username: admin - password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 - secretKey: somesecretvalue -name: harbor -metadata: - name: harbor diff --git a/tests/fixtures/env/apps/secrets.keycloak.yaml b/tests/fixtures/env/apps/secrets.keycloak.yaml deleted file mode 100644 index 34f328f775..0000000000 --- a/tests/fixtures/env/apps/secrets.keycloak.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - idp: - clientSecret: somsecretvalue -name: keycloak -metadata: - name: keycloak diff --git a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml b/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml deleted file mode 100644 index 953af4f5c9..0000000000 --- a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - rootPassword: mysqlsomesecretvalue -name: kubeflow-pipelines -metadata: - name: kubeflow-pipelines diff --git a/tests/fixtures/env/apps/secrets.loki.yaml b/tests/fixtures/env/apps/secrets.loki.yaml deleted file mode 100644 index fd1fbcf597..0000000000 --- a/tests/fixtures/env/apps/secrets.loki.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: loki -metadata: - name: loki diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml deleted file mode 100644 index 56f2d4ce60..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - password: gkhugxJsPjhbCybH -name: oauth2-proxy-redis -metadata: - name: oauth2-proxy-redis diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml deleted file mode 100644 index 9361e610fe..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - config: - cookieSecret: gkhugxJsPjhbCybH -name: oauth2-proxy -metadata: - name: oauth2-proxy diff --git a/tests/fixtures/env/apps/secrets.prometheus.yaml b/tests/fixtures/env/apps/secrets.prometheus.yaml deleted file mode 100644 index 74b9528e64..0000000000 --- a/tests/fixtures/env/apps/secrets.prometheus.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplApp -spec: - remoteWrite: - rwConfig: - basicAuth: - password: blalalalalal -name: prometheus -metadata: - name: prometheus diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml new file mode 100644 index 0000000000..ad4e1b66c5 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: alerts-secrets + namespace: apl-secrets +spec: + encryptedData: + slack_url: https://hooks.slack.com/services/id + msteams_highPrio: https://xxxxxxx.com + msteams_lowPrio: https://xxxxxxxx.com + template: + immutable: false + metadata: + name: alerts-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml new file mode 100644 index 0000000000..e9e5f54146 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/argocd-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: argocd-secrets + namespace: apl-secrets +spec: + encryptedData: + redisPassword: someEncryptedPassword + template: + immutable: false + metadata: + name: argocd-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml new file mode 100644 index 0000000000..0dd9c28078 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml @@ -0,0 +1,43 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: cert-manager-secrets + namespace: apl-secrets +spec: + encryptedData: + customRootCAKey: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ + ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd + BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K + GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq + pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH + NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 + 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj + VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr + V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT + UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN + joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B + EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ + AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE + 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT + MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw + UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa + DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl + BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM + wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC + bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ + E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K + 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK + CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H + rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX + EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= + -----END RSA PRIVATE KEY----- + template: + immutable: false + metadata: + name: cert-manager-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml new file mode 100644 index 0000000000..fc1b043bc6 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: dns-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx + template: + immutable: false + metadata: + name: dns-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml new file mode 100644 index 0000000000..f9c85ad94d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: gitea-secrets + namespace: apl-secrets +spec: + encryptedData: + postgresqlPassword: postgresqlPassword + template: + immutable: false + metadata: + name: gitea-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml new file mode 100644 index 0000000000..4138e88e3b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: grafana-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: grafana-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml new file mode 100644 index 0000000000..60b72c8778 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml @@ -0,0 +1,24 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: harbor-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: harborsomesecretvalue + core_secret: vQFMm9Qk0pTUF3MK + core_xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi + jobservice_secret: CfpanIkcGWz3wGLO + registry_secret: PjHGEdmPhrmNrEkj + registry_credentials_htpasswd: "admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6" + registry_credentials_username: admin + registry_credentials_password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 + secretKey: somesecretvalue + template: + immutable: false + metadata: + name: harbor-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml new file mode 100644 index 0000000000..9f1dac4209 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: keycloak-secrets + namespace: apl-secrets +spec: + encryptedData: + idp_clientSecret: somsecretvalue + template: + immutable: false + metadata: + name: keycloak-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml new file mode 100644 index 0000000000..949018981b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: kubeflow-pipelines-secrets + namespace: apl-secrets +spec: + encryptedData: + rootPassword: mysqlsomesecretvalue + template: + immutable: false + metadata: + name: kubeflow-pipelines-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml new file mode 100644 index 0000000000..713eb67bd7 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: loki-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: loki-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml new file mode 100644 index 0000000000..0e0e823755 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-redis-secrets + namespace: apl-secrets +spec: + encryptedData: + password: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-redis-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml new file mode 100644 index 0000000000..b47b028e1b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-secrets + namespace: apl-secrets +spec: + encryptedData: + config_cookieSecret: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml new file mode 100644 index 0000000000..e8e1f6336f --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: obj-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_secretAccessKey: somesecretvalue + template: + immutable: false + metadata: + name: obj-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml new file mode 100644 index 0000000000..b764c3fd6d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oidc-secrets + namespace: apl-secrets +spec: + encryptedData: + clientSecret: somesecretvalue + template: + immutable: false + metadata: + name: oidc-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml new file mode 100644 index 0000000000..2c805cb155 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: otomi-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: bladibla + git_password: gitPasswordForTesting + globalPullSecret_password: blablabla + template: + immutable: false + metadata: + name: otomi-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml new file mode 100644 index 0000000000..d8589306cd --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: prometheus-secrets + namespace: apl-secrets +spec: + encryptedData: + remoteWrite_rwConfig_basicAuth_password: blalalalalal + template: + immutable: false + metadata: + name: prometheus-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml new file mode 100644 index 0000000000..b03b2ab05e --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: smtp-secrets + namespace: apl-secrets +spec: + encryptedData: + auth_password: somesecretvalue + template: + immutable: false + metadata: + name: smtp-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml new file mode 100644 index 0000000000..971824fd93 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-admin-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: YTrnkdUsKPcGATfg + template: + immutable: false + metadata: + name: team-admin-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml new file mode 100644 index 0000000000..214aa453c3 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-demo-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: somesecretvalue + template: + immutable: false + metadata: + name: team-demo-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml new file mode 100644 index 0000000000..fd3676c286 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-dev-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: IkdUsKPcGAdanjas + template: + immutable: false + metadata: + name: team-dev-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml new file mode 100644 index 0000000000..cd965eb983 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users +spec: + encryptedData: + email: team@admin.com + firstName: team + lastName: admin + initialPassword: team-admin-password + isPlatformAdmin: 'false' + isTeamAdmin: 'true' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml new file mode 100644 index 0000000000..a9cabecfdb --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users +spec: + encryptedData: + email: team@member.com + firstName: team + lastName: member + initialPassword: team-member-password + isPlatformAdmin: 'false' + isTeamAdmin: 'false' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml new file mode 100644 index 0000000000..1ae170984a --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users +spec: + encryptedData: + email: platform@admin.com + firstName: platform + lastName: admin + initialPassword: platform-admin-password + isPlatformAdmin: 'true' + isTeamAdmin: 'true' + teams: '[]' + template: + immutable: false + metadata: + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/settings/kms.yaml b/tests/fixtures/env/settings/kms.yaml index 70e04a7c17..f3dd212daf 100644 --- a/tests/fixtures/env/settings/kms.yaml +++ b/tests/fixtures/env/settings/kms.yaml @@ -6,5 +6,6 @@ spec: sops: azure: clientId: somesecretvalue + clientSecret: somesecretvalue keys: somesecretvalue tenantId: somesecretvalue diff --git a/tests/fixtures/env/settings/secrets.alerts.yaml b/tests/fixtures/env/settings/secrets.alerts.yaml deleted file mode 100644 index 1bbbfdaf12..0000000000 --- a/tests/fixtures/env/settings/secrets.alerts.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplAlertSet -spec: - slack: - url: https://hooks.slack.com/services/id - msteams: - highPrio: https://xxxxxxx.com - lowPrio: https://xxxxxxxx.com -name: alerts -metadata: - name: alerts diff --git a/tests/fixtures/env/settings/secrets.dns.yaml b/tests/fixtures/env/settings/secrets.dns.yaml deleted file mode 100644 index a6024d2e07..0000000000 --- a/tests/fixtures/env/settings/secrets.dns.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplDns -spec: - provider: - linode: - apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx -name: dns -metadata: - name: dns diff --git a/tests/fixtures/env/settings/secrets.kms.yaml b/tests/fixtures/env/settings/secrets.kms.yaml deleted file mode 100644 index 74b33f1f1c..0000000000 --- a/tests/fixtures/env/settings/secrets.kms.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplKms -spec: - sops: - azure: - clientSecret: somesecretvalue -name: kms -metadata: - name: kms diff --git a/tests/fixtures/env/settings/secrets.obj.yaml b/tests/fixtures/env/settings/secrets.obj.yaml deleted file mode 100644 index 82c77762ff..0000000000 --- a/tests/fixtures/env/settings/secrets.obj.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplObjectStorage -spec: - provider: - linode: - secretAccessKey: somesecretvalue -name: obj -metadata: - name: obj diff --git a/tests/fixtures/env/settings/secrets.oidc.yaml b/tests/fixtures/env/settings/secrets.oidc.yaml deleted file mode 100644 index 5f03c2f11d..0000000000 --- a/tests/fixtures/env/settings/secrets.oidc.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplIdentityProvider -spec: - clientSecret: somesecretvalue -name: oidc -metadata: - name: oidc diff --git a/tests/fixtures/env/settings/secrets.otomi.yaml b/tests/fixtures/env/settings/secrets.otomi.yaml deleted file mode 100644 index 1819165b7c..0000000000 --- a/tests/fixtures/env/settings/secrets.otomi.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplCapabilitySet -spec: - adminPassword: bladibla - git: - password: gitPasswordForTesting - globalPullSecret: - password: blablabla -name: otomi -metadata: - name: otomi diff --git a/tests/fixtures/env/settings/secrets.smtp.yaml b/tests/fixtures/env/settings/secrets.smtp.yaml deleted file mode 100644 index 62a1d33e38..0000000000 --- a/tests/fixtures/env/settings/secrets.smtp.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplSmtp -spec: - auth_password: somesecretvalue -name: smtp -metadata: - name: smtp diff --git a/tests/fixtures/env/settings/versions.yaml b/tests/fixtures/env/settings/versions.yaml index 127f1b82c6..dac897d77c 100644 --- a/tests/fixtures/env/settings/versions.yaml +++ b/tests/fixtures/env/settings/versions.yaml @@ -3,4 +3,4 @@ metadata: name: versions labels: {} spec: - specVersion: 60 + specVersion: 61 diff --git a/tests/fixtures/env/teams/admin/secrets.settings.yaml b/tests/fixtures/env/teams/admin/secrets.settings.yaml deleted file mode 100644 index 595f37ad22..0000000000 --- a/tests/fixtures/env/teams/admin/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: YTrnkdUsKPcGATfg -name: admin -metadata: - name: admin diff --git a/tests/fixtures/env/teams/demo/secrets.settings.yaml b/tests/fixtures/env/teams/demo/secrets.settings.yaml deleted file mode 100644 index abf4440deb..0000000000 --- a/tests/fixtures/env/teams/demo/secrets.settings.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: somesecretvalue - alerts: - slack: - url: https://slack.con -name: demo -metadata: - name: demo diff --git a/tests/fixtures/env/teams/dev/secrets.settings.yaml b/tests/fixtures/env/teams/dev/secrets.settings.yaml deleted file mode 100644 index 41599125ef..0000000000 --- a/tests/fixtures/env/teams/dev/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: IkdUsKPcGAdanjas -name: dev -metadata: - name: dev diff --git a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml deleted file mode 100644 index 6576468c59..0000000000 --- a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 23d63558-49ed-48ba-bc28-8037a7236ddf -spec: - email: team@admin.com - firstName: team - initialPassword: team-admin-password - isPlatformAdmin: false - isTeamAdmin: true - lastName: admin - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml deleted file mode 100644 index 839049b33f..0000000000 --- a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 -spec: - email: team@member.com - firstName: team - initialPassword: team-member-password - isPlatformAdmin: false - isTeamAdmin: false - lastName: member - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml deleted file mode 100644 index a489101d4f..0000000000 --- a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: a83e20b7-474a-4262-a3ad-b09813364ece -spec: - email: platform@admin.com - firstName: platform - initialPassword: platform-admin-password - isPlatformAdmin: true - isTeamAdmin: true - lastName: admin diff --git a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml deleted file mode 100644 index e0fc2601d6..0000000000 --- a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 -spec: - email: platform-admin@dev.linode-apl.net - firstName: platform - lastName: admin - isPlatformAdmin: true - isTeamAdmin: false - initialPassword: 02LDWB#qzknkeF8f*m%% diff --git a/values-changes.yaml b/values-changes.yaml index b3b08cb2de..297c99d38c 100644 --- a/values-changes.yaml +++ b/values-changes.yaml @@ -461,3 +461,8 @@ changes: - version: 60 customFunctions: - addRedisSecretForArgoCD + - version: 61 + deletions: + - 'kms.sops' + customFunctions: + - sopsMigration diff --git a/values-schema.yaml b/values-schema.yaml index 2176f3b51e..9a2c1e058e 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -923,7 +923,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - x-secret: 'admin' + x-secret: 'otomi-admin' securityContext: additionalProperties: uniqueItems: true @@ -1581,6 +1581,31 @@ properties: - recovery - standard default: standard + recovery: + type: object + description: Recovery settings for disaster recovery scenarios. Only used when mode=recovery. + properties: + manifests: + type: object + description: > + K8s resource List containing recovery manifests (e.g., sealed-secrets TLS key pairs) + to apply during installation. Not stored in the values repository. + Export from old cluster: + kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key=active -o yaml + properties: + apiVersion: + type: string + kind: + type: string + items: + type: array + items: + type: object + additionalProperties: true + metadata: + type: object + additionalProperties: true + additionalProperties: true azure: description: Azure specific configuration. properties: @@ -1684,7 +1709,6 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: - x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: @@ -1744,6 +1768,12 @@ properties: $ref: '#/definitions/rawValues' enabled: type: boolean + adminUsername: + type: string + default: otomi-admin + adminPassword: + type: string + x-secret: '{{ randAlphaNum 20 }}' postgresqlPassword: type: string description: This password was generated and cannot be changed without manual intervention. @@ -1831,7 +1861,6 @@ properties: type: string x-secret: '{{ randAlphaNum 32 }}' required: - - secret - credentials databaseMaxConnections: type: number @@ -1939,6 +1968,7 @@ properties: x-secret: '' adminUsername: type: string + default: otomi-admin theme: type: string default: otomi @@ -2786,6 +2816,7 @@ properties: $ref: '#/definitions/url' clientID: $ref: '#/definitions/wordCharacterPattern' + x-secret: '' clientSecret: type: string x-secret: '' @@ -2805,7 +2836,6 @@ properties: default: sub required: - clientID - - clientSecret - issuer otomi: additionalProperties: false diff --git a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl index 36fa0f144a..d6b6c4c167 100644 --- a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl +++ b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl @@ -19,13 +19,32 @@ resources: hasArgocd: "{{ $v.apps.argocd.enabled | toString }}" domainSuffix: "{{ $v.cluster.domainSuffix }}" teamConfig: {{ $teamConfig | toJson }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-gitea-operator-secret namespace: apl-gitea-operator - data: - giteaPassword: {{ $v.otomi.git.password | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-gitea-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + giteaPassword: '{{ "{{ .gitPassword | toString }}" }}' + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + data: + - secretKey: gitPassword + remoteRef: + key: otomi-secrets + property: git_password + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 1a58d1fec9..265e5058e0 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -11,17 +11,36 @@ {{- $teamNamespaces := $teamNamespaces | sortAlpha | toJson }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-harbor-operator-secret namespace: apl-harbor-operator - data: - harborPassword: {{ $h.adminPassword | b64enc }} - harborUser: {{ "admin" | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-harbor-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + harborPassword: '{{ "{{ .harborAdminPassword | toString }}" }}' + harborUser: admin + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + data: + - secretKey: harborAdminPassword + remoteRef: + key: otomi-secrets + property: adminPassword + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 5a3a8609c4..b07da67e9b 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -21,31 +21,49 @@ {{- $k := $c | get "keycloak" }} {{- $doms := tpl (readFile "../../helmfile.d/snippets/domains.gotmpl") $v | fromYaml }} {{- $joinTpl := readFile "../../helmfile.d/utils/joinListWithSep.gotmpl" }} -{{ $users := list }} -{{- range $user := $v.users }} - {{ $groups := list }} - {{- if $user.isPlatformAdmin }}{{ $groups = append $groups "platform-admin" }}{{ end }} - {{- if $user.isTeamAdmin }}{{ $groups = append $groups "team-admin" }}{{ end }} - {{- range $team := $user | get "teams" list }}{{ $groups = append $groups (print "team-" $team) }}{{ end }} - {{- $users = append $users (dict "email" $user.email "firstName" $user.firstName "lastName" $user.lastName "initialPassword" $user.initialPassword "groups" $groups) }} -{{- end }} -{{- $users := $users | toJson }} - resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-keycloak-operator-secret namespace: apl-keycloak-operator - data: - KEYCLOAK_ADMIN: {{ .Values.apps.keycloak.adminUsername | b64enc }} - KEYCLOAK_ADMIN_PASSWORD: {{ $k.adminPassword | b64enc }} - KEYCLOAK_CLIENT_SECRET: {{ $k.idp.clientSecret | b64enc }} - USERS: {{ $users | b64enc }} - {{- if $v.otomi.hasExternalIDP }} - IDP_CLIENT_ID: {{ $oi.clientID | b64enc}} - IDP_CLIENT_SECRET: {{ $oi.clientSecret | b64enc }} - {{- end }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-keycloak-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + KEYCLOAK_ADMIN: {{ $k.adminUsername }} + KEYCLOAK_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + KEYCLOAK_CLIENT_SECRET: '{{ "{{ .idpClientSecret | toString }}" }}' + {{- if $v.otomi.hasExternalIDP }} + IDP_CLIENT_ID: '{{ "{{ .oidcClientID | toString }}" }}' + IDP_CLIENT_SECRET: '{{ "{{ .oidcClientSecret | toString }}" }}' + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-secrets + property: adminPassword + - secretKey: idpClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + {{- if $v.otomi.hasExternalIDP }} + - secretKey: oidcClientID + remoteRef: + key: oidc-secrets + property: clientID + - secretKey: oidcClientSecret + remoteRef: + key: oidc-secrets + property: clientSecret + {{- end }} - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl new file mode 100644 index 0000000000..41228f7257 --- /dev/null +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -0,0 +1,25 @@ +{{- $v := .Values }} + +resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: apl-git-credentials + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-git-credentials + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $v.otomi.git | get "username" "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-secrets + property: git_password diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index d3bc151b2a..a54c848fa6 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -2,7 +2,6 @@ {{- $o := $v.apps | get "apl-operator" }} {{- $version := $v.otomi.version }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} -{{- $g := $v.apps.gitea }} {{- $kms := $v | get "kms" dict }} image: {{- if $v.otomi.linodeLkeImageRepository }} @@ -21,9 +20,9 @@ resources: {{- toYaml $o.resources.operator | nindent 2 }} kms: {{- $kms | toYaml | nindent 2 }} git: - password: {{ $v.otomi.git.password | quote }} - username: {{ $v.otomi.git.username | quote }} - email: {{ $v.otomi.git.email | quote }} repoUrl: {{ $v.otomi.git.repoUrl | quote }} - branch: {{ $v.otomi.git.branch | quote }} - + branch: {{ $v.otomi.git.branch | default "main" | quote }} + email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} + username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} + # Password intentionally empty — operator reads git credentials from K8s secret (apl-git-credentials) + password: "" diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 0af4be4949..9165539c0d 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -10,51 +10,131 @@ resources: custom-ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} {{- if contains "gitea-http.gitea.svc.cluster.local" $v.otomi.git.repoUrl }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ printf "https://%s" $v._derived.giteaDomain | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} - - apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: git + url: {{ printf "https://%s" $v._derived.giteaDomain }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-secrets + property: git_password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea-internal namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | b64enc | quote }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea-internal + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: git + url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | quote }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-secrets + property: git_password {{- else }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-git namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ $v.otomi.git.repoUrl | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-git + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: git + url: {{ $v.otomi.git.repoUrl }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-secrets + property: git_password {{- end }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - name: argocd-redis + name: argocd-oidc-secret namespace: argocd - data: - auth: {{ $a.redisPassword | b64enc | quote }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-secret + creationPolicy: Merge + data: + - secretKey: oidc.clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: argocd-redis-secret + namespace: argocd + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-redis + creationPolicy: Merge + template: + data: + auth: '{{ "{{ .redisPassword | toString }}" }}' + data: + - secretKey: redisPassword + remoteRef: + key: argocd-secrets + property: redisPassword - apiVersion: v1 kind: Secret metadata: diff --git a/values/argocd/argocd.gotmpl b/values/argocd/argocd.gotmpl index 2684df9a32..9f892876a1 100644 --- a/values/argocd/argocd.gotmpl +++ b/values/argocd/argocd.gotmpl @@ -226,8 +226,7 @@ configs: {{- end }} secret: - extra: - oidc.clientSecret: {{ $k.idp.clientSecret | quote }} + extra: {} params: server.insecure: true # nginx terminates tls # -- Number of application status processors diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index 6f7b113888..af97af4995 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -6,30 +6,98 @@ resources: {{- if and $v.otomi.hasExternalDNS (or (not (hasKey $p "aws")) ($p | get "aws.credentials.secretKey" nil)) }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - name: "external-dns" - data: - {{- if hasKey $p "google" }} - secret: "{{ $p.google.serviceAccountKey | b64enc }}" - {{- else if hasKey $p "akamai" }} - access_token: {{ $p.akamai.accessToken | b64enc | quote }} - client_token: {{ $p.akamai.clientToken | b64enc | quote }} - client_secret: {{ $p.akamai.clientSecret | b64enc | quote }} - {{- else if hasKey $p "azure-private-dns" }} - secret: "{{ $p | get "azure-private-dns.aadClientSecret" | b64enc }}" - {{- else if hasKey $p "azure" }} - secret: "{{ $p.azure.aadClientSecret | b64enc }}" - {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} - secret: "{{ $p.aws.credentials.secretKey | b64enc }}" - {{- else if hasKey $p "digitalocean" }} - secret: "{{ $p.digitalocean.apiToken | b64enc }}" - {{- else if hasKey $p "cloudflare" }} - secret: "{{ $p.cloudflare.apiToken | b64enc }}" - {{- else if hasKey $p "linode" }} - secret: "{{ $p.linode.apiToken | b64enc }}" - {{- end }} + name: external-dns + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: external-dns + creationPolicy: Owner + template: + type: Opaque + data: + {{- if hasKey $p "google" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "akamai" }} + access_token: '{{ "{{ .access_token | toString }}" }}' + client_token: '{{ "{{ .client_token | toString }}" }}' + client_secret: '{{ "{{ .client_secret | toString }}" }}' + {{- else if hasKey $p "azure-private-dns" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "azure" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "digitalocean" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "cloudflare" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "linode" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- end }} + data: + {{- if hasKey $p "akamai" }} + - secretKey: access_token + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken + - secretKey: client_token + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: client_secret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + {{- else }} + - secretKey: secret + remoteRef: + key: dns-secrets + {{- if hasKey $p "google" }} + property: provider_google_serviceAccountKey + {{- else if hasKey $p "azure-private-dns" }} + property: provider_azure-private-dns_aadClientSecret + {{- else if hasKey $p "azure" }} + property: provider_azure_aadClientSecret + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + property: provider_aws_credentials_secretKey + {{- else if hasKey $p "digitalocean" }} + property: provider_digitalocean_apiToken + {{- else if hasKey $p "cloudflare" }} + property: provider_cloudflare_apiToken + {{- else if hasKey $p "linode" }} + property: provider_linode_apiToken + {{- end }} + {{- end }} +{{- end }} +{{- if $cm | get "customRootCA" "" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: custom-ca + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: custom-ca + creationPolicy: Owner + template: + type: Opaque + data: + tls.crt: {{ $cm.customRootCA | quote }} + tls.key: '{{ "{{ .customRootCAKey | toString }}" }}' + data: + - secretKey: customRootCAKey + remoteRef: + key: cert-manager-secrets + property: customRootCAKey {{- end }} - apiVersion: cert-manager.io/v1 kind: ClusterIssuer @@ -147,15 +215,29 @@ resources: {{- end }} {{- end }} {{- if eq $cm.issuer "byo-wildcard-cert" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-byo-wildcard-cert namespace: istio-system - type: kubernetes.io/tls - data: - tls.crt: {{ $cm.byoWildcardCert | b64enc }} - tls.key: {{ $cm.byoWildcardCertKey | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-byo-wildcard-cert + creationPolicy: Owner + template: + type: kubernetes.io/tls + data: + tls.crt: {{ $cm.byoWildcardCert | quote }} + tls.key: '{{ "{{ .byoWildcardCertKey | toString }}" }}' + data: + - secretKey: byoWildcardCertKey + remoteRef: + key: cert-manager-secrets + property: byoWildcardCertKey {{- end }} {{- if or (eq $cm.issuer "letsencrypt" ) (eq $cm.issuer "custom-ca" ) }} - apiVersion: cert-manager.io/v1 @@ -179,4 +261,4 @@ resources: - key encipherment - ocsp signing {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/values/external-dns/external-dns-raw.gotmpl b/values/external-dns/external-dns-raw.gotmpl index e70774372c..9f699a9c1a 100644 --- a/values/external-dns/external-dns-raw.gotmpl +++ b/values/external-dns/external-dns-raw.gotmpl @@ -3,103 +3,252 @@ {{- with $d.provider | get "akamai" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: akamai-dns - data: - EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: "{{.clientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: akamai-dns + creationPolicy: Owner + template: + type: Opaque + data: + EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: '{{ "{{ .clientSecret | toString }}" }}' + EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN: '{{ "{{ .clientToken | toString }}" }}' + EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN: '{{ "{{ .accessToken | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + - secretKey: clientToken + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: accessToken + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken {{- end }} {{- with $d.provider | get "linode" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-dns-api-token - data: - LINODE_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-dns-api-token + creationPolicy: Owner + template: + type: Opaque + data: + LINODE_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_linode_apiToken {{- end }} {{- with $d.provider | get "digitalocean" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: do-token - data: - DO_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: do-token + creationPolicy: Owner + template: + type: Opaque + data: + DO_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_digitalocean_apiToken {{- end }} {{- with $d.provider | get "cloudflare" nil }} {{- with .apiToken }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_TOKEN: "{{ . | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiToken {{- end }} {{- with .apiSecret }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_KEY: "{{ . | b64enc }}" - CF_API_EMAIL: "{{ $d.provider.cloudflare.email | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_KEY: '{{ "{{ .apiSecret | toString }}" }}' + CF_API_EMAIL: {{ $d.provider.cloudflare.email | quote }} + data: + - secretKey: apiSecret + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiSecret {{- end }} {{- end }} {{- with $d.provider | get "aws" nil }} {{- with .credentials }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: aws-dns-credentials - data: - AWS_ACCESS_KEY_ID: "{{ .accessKey | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ .secretKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: aws-dns-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: '{{ "{{ .accessKey | toString }}" }}' + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: accessKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_accessKey + - secretKey: secretKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_secretKey {{- end }} {{- end }} {{- with $d.provider | get "azure" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure_aadClientSecret {{- end }} {{- with $d.provider | get "azure-private-dns" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-private-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-private-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure-private-dns_aadClientSecret {{- end }} {{- with $d.provider | get "google" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: google-dns - data: - GOOGLE_APPLICATION_CREDENTIALS: "{{ .serviceAccountKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: google-dns + creationPolicy: Owner + template: + type: Opaque + data: + GOOGLE_APPLICATION_CREDENTIALS: '{{ "{{ .serviceAccountKey | toString }}" }}' + data: + - secretKey: serviceAccountKey + remoteRef: + key: dns-secrets + property: provider_google_serviceAccountKey {{- end }} diff --git a/values/external-dns/external-dns.gotmpl b/values/external-dns/external-dns.gotmpl index faa85e2f0f..a1be069c9a 100644 --- a/values/external-dns/external-dns.gotmpl +++ b/values/external-dns/external-dns.gotmpl @@ -50,9 +50,15 @@ env: - name: EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN value: "{{ $dns.provider.akamai.host }}" - name: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - value: "{{ $dns.provider.akamai.clientToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - name: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - value: "{{ $dns.provider.akamai.accessToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - name: EXTERNAL_DNS_AKAMAI_CLIENT_SECRET valueFrom: secretKeyRef: @@ -208,7 +214,7 @@ env: extraVolumes: - name: google-service-account secret: - secretName: GOOGLE-DNS + secretName: google-dns extraVolumeMounts: - name: google-service-account mountPath: /etc/secrets/service-account diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl new file mode 100644 index 0000000000..c1b95b3111 --- /dev/null +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -0,0 +1,47 @@ +{{- $v := .Values }} + +resources: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: eso-store-sa + namespace: external-secrets + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: eso-core-secret-reader + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: eso-core-secret-reader-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eso-core-secret-reader + subjects: + - kind: ServiceAccount + name: eso-store-sa + namespace: external-secrets + - apiVersion: external-secrets.io/v1beta1 + kind: ClusterSecretStore + metadata: + name: core-secrets-store + spec: + provider: + kubernetes: + remoteNamespace: apl-secrets + server: + url: "https://kubernetes.default.svc" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + namespace: external-secrets + key: ca.crt + auth: + serviceAccount: + name: eso-store-sa + namespace: external-secrets diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl new file mode 100644 index 0000000000..0e16bcfaaf --- /dev/null +++ b/values/external-secrets/external-secrets.gotmpl @@ -0,0 +1,9 @@ +{{- $v := .Values }} +{{- $app := $v.apps | get "external-secrets" }} + +replicaCount: 1 + +resources: {{- $app.resources.operator | toYaml | nindent 2 }} + +image: + repository: ghcr.io/external-secrets/external-secrets diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 792c355e5f..7f2b9e4f2a 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -1,12 +1,25 @@ {{- $v := .Values }} -{{- $g := $v.apps.gitea }} resources: -- apiVersion: v1 - kind: Secret - type: kubernetes.io/basic-auth +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: gitea-db-secret - data: - username: "{{ "gitea" | b64enc }}" - password: "{{ $g.postgresqlPassword | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-db-secret + creationPolicy: Owner + template: + type: kubernetes.io/basic-auth + data: + username: {{ $v._derived.giteaDbUsername }} + password: '{{ "{{ .postgresqlPassword | toString }}" }}' + data: + - secretKey: postgresqlPassword + remoteRef: + key: gitea-secrets + property: postgresqlPassword diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 02b2cc15e8..d7d84c04a4 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -13,6 +13,28 @@ {{- end }} resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-admin-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $v.otomi.git.username | default "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-secrets + property: git_password {{- if $v._derived.untrustedCA }} - apiVersion: v1 kind: Secret @@ -21,22 +43,53 @@ resources: data: ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} -- apiVersion: v1 - kind: Secret - metadata: - name: gitea-admin-secret - data: - username: "{{ $v.otomi.git.username | b64enc }}" - password: "{{ $v.otomi.git.password | b64enc }}" # DB / app backup resources {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey +{{- end }} +{{- with $v | get "smtp" nil }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-smtp-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-smtp-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .authPassword | toString }}" }}' + data: + - secretKey: authPassword + remoteRef: + key: smtp-secrets + property: auth_password {{- end }} {{- if ne $v.cluster.provider "custom" }} # Application backup resources diff --git a/values/gitea/gitea.gotmpl b/values/gitea/gitea.gotmpl index a77a4ebeea..48f73da9a8 100644 --- a/values/gitea/gitea.gotmpl +++ b/values/gitea/gitea.gotmpl @@ -64,7 +64,7 @@ gitea: HELO_HOSTNAME: {{ .hello }} FROM: {{ .from }} USER: {{ .auth_username }} - PASSWD: {{ .auth_password | quote }} + PASSWD: placeholder-overridden-by-env MAILER_TYPE: smtp IS_TLS_ENABLED: true SUBJECT_PREFIX: 'Otomi[Gitea]: ' @@ -126,6 +126,13 @@ gitea: secretKeyRef: name: gitea-db-secret key: password + {{- with $v | get "smtp" nil }} + - name: GITEA__MAILER__PASSWD + valueFrom: + secretKeyRef: + name: gitea-smtp-secret + key: password + {{- end }} - name: GITEA__DATABASE__MAX_OPEN_CONNS value: {{ $g.databaseMaxConnections | quote }} - name: GITEA__DATABASE__MAX_IDLE_CONNS diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index a781deccb0..a6d199dc1a 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -34,60 +34,208 @@ resources: issuerRef: name: custom-ca kind: ClusterIssuer -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-admin-password - data: - HARBOR_ADMIN_PASSWORD: "{{ $h.adminPassword | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-admin-password + creationPolicy: Owner + template: + type: Opaque + data: + HARBOR_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: otomi-secrets + property: adminPassword +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-credentials - data: - REGISTRY_PASSWD: "{{ $h.registry.credentials.password | b64enc }}" - REGISTRY_HTPASSWD: "{{ $h.registry.credentials.htpasswd | b64enc }}" -{{- if ne $h.secretKey nil }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_PASSWD: '{{ "{{ .password | toString }}" }}' + REGISTRY_HTPASSWD: '{{ "{{ .htpasswd | toString }}" }}' + REGISTRY_USERNAME: '{{ "{{ .username | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: harbor-secrets + property: registry_credentials_password + - secretKey: htpasswd + remoteRef: + key: harbor-secrets + property: registry_credentials_htpasswd + - secretKey: username + remoteRef: + key: harbor-secrets + property: registry_credentials_username +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-secret-key - data: - secretKey: "{{ $h.secretKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-secret-key + creationPolicy: Owner + template: + type: Opaque + data: + secretKey: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: secretKey + remoteRef: + key: harbor-secrets + property: secretKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-core-secret - data: - secret: "{{ $h.core.secret | b64enc }}" -{{- end }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-secret + creationPolicy: Owner + template: + type: Opaque + data: + secret: '{{ "{{ .coreSecret | toString }}" }}' + data: + - secretKey: coreSecret + remoteRef: + key: harbor-secrets + property: core_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: harbor-core-xsrf-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-xsrf-secret + creationPolicy: Owner + template: + type: Opaque + data: + CSRF_KEY: '{{ "{{ .xsrfKey | toString }}" }}' + data: + - secretKey: xsrfKey + remoteRef: + key: harbor-secrets + property: core_xsrfKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-jobservice-secret - data: - JOBSERVICE_SECRET: "{{ $h.jobservice.secret | default "" | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-jobservice-secret + creationPolicy: Owner + template: + type: Opaque + data: + JOBSERVICE_SECRET: '{{ "{{ .jobserviceSecret | toString }}" }}' + data: + - secretKey: jobserviceSecret + remoteRef: + key: harbor-secrets + property: jobservice_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-http - data: - REGISTRY_HTTP_SECRET: "{{ $h.registry.secret | default "" | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-http + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_HTTP_SECRET: '{{ "{{ .registrySecret | toString }}" }}' + data: + - secretKey: registrySecret + remoteRef: + key: harbor-secrets + property: registry_secret {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: registry-storage-credentials - data: - REGISTRY_STORAGE_S3_ACCESSKEY: "{{ $obj.linode.accessKeyId | b64enc }}" - REGISTRY_STORAGE_S3_SECRETKEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: registry-storage-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_STORAGE_S3_ACCESSKEY: {{ $obj.linode.accessKeyId }} + REGISTRY_STORAGE_S3_SECRETKEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey {{- end }} - apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute diff --git a/values/harbor/harbor.gotmpl b/values/harbor/harbor.gotmpl index a623af0823..c871d9e649 100644 --- a/values/harbor/harbor.gotmpl +++ b/values/harbor/harbor.gotmpl @@ -26,7 +26,14 @@ core: resources: {{- $h.resources.core | toYaml | nindent 4 }} existingSecret: harbor-core-secret - xsrfKey: {{ $h | get "core.xsrfKey" nil }} + existingXsrfSecret: harbor-core-xsrf-secret + existingXsrfSecretKey: CSRF_KEY + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME database: maxOpenConns: {{ $h.databaseMaxConnections }} @@ -72,6 +79,12 @@ jobservice: - stdout resources: {{- $h.resources.jobservice | toYaml | nindent 4 }} existingSecret: harbor-jobservice-secret + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME metrics: serviceMonitor: diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 28e89c9963..7ee4eef902 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -7,7 +7,7 @@ resources: labels: app.kubernetes.io/component: controller name: {{ $ingress.className }} - {{- if eq $ingress.className $v.ingress.platformClass.className }} + {{- if eq $ingress.className $v.ingress.platformClass.className }} annotations: ingressclass.kubernetes.io/is-default-class: "true" {{- end }} diff --git a/values/k8s/k8s-raw-teams.gotmpl b/values/k8s/k8s-raw-teams.gotmpl index c647f9685f..83fdbd8fa0 100644 --- a/values/k8s/k8s-raw-teams.gotmpl +++ b/values/k8s/k8s-raw-teams.gotmpl @@ -1,6 +1,5 @@ {{- $v := .Values }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $dockerConfigTpl := readFile "../../helmfile.d/snippets/dockercfg.gotmpl" }} resources: {{- range $id, $team := omit $v.teamConfig "admin"}} {{- $ns := printf "team-%s" $id }} @@ -17,14 +16,31 @@ resources: istio-injection: enabled {{- end }} {{- with $v.otomi | get "globalPullSecret" nil }} - - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson + {{- $gpsUsername := . | get "username" "" }} + {{- $gpsServer := . | get "server" "docker.io" }} + {{- $gpsEmail := . | get "email" "not@val.id" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-pullsecret-global namespace: {{ $ns }} - data: - .dockerconfigjson: {{ tpl $dockerConfigTpl . | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-pullsecret-global + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' + data: + - secretKey: password + remoteRef: + key: otomi-secrets + property: globalPullSecret_password {{- end }} # patching service account here as helm does not recognize it as it's own - apiVersion: v1 diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index eb25208035..f0e39c4c0b 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -1,6 +1,5 @@ {{- $v := .Values }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $dockerConfigTpl := readFile "../../helmfile.d/snippets/dockercfg.gotmpl" }} resources: {{- if ( (hasKey $v "bootstrap"))}} - {{- $v.bootstrap | toYaml | nindent 4 }} @@ -21,14 +20,31 @@ resources: {{- end }} {{- with $ns | get "labels" nil }}{{ toYaml . | nindent 8 }}{{ end }} {{- with $v.otomi | get "globalPullSecret" nil }} - - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson + {{- $gpsUsername := . | get "username" "" }} + {{- $gpsServer := . | get "server" "docker.io" }} + {{- $gpsEmail := . | get "email" "not@val.id" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-pullsecret-global namespace: {{ $ns.name }} - data: - .dockerconfigjson: {{ tpl $dockerConfigTpl . | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-pullsecret-global + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' + data: + - secretKey: password + remoteRef: + key: otomi-secrets + property: globalPullSecret_password {{- end }} - apiVersion: v1 kind: ServiceAccount @@ -72,12 +88,3 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." - - - apiVersion: v1 - kind: Secret - metadata: - name: custom-ca - namespace: cert-manager - data: - tls.crt: {{ $cm.customRootCA | b64enc }} - tls.key: {{ $cm.customRootCAKey | b64enc}} diff --git a/values/keycloak/keycloak-raw.gotmpl b/values/keycloak/keycloak-raw.gotmpl index 3e483fd345..ef85209a5b 100644 --- a/values/keycloak/keycloak-raw.gotmpl +++ b/values/keycloak/keycloak-raw.gotmpl @@ -1,5 +1,5 @@ {{- $v := .Values }} -{{- $otomiAdmin := "otomi-admin" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} resources: @@ -9,20 +9,50 @@ resources: name: custom-ca data: custom-ca.pem: {{ .Values._derived.caCert | b64enc }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: keycloak-initial-admin namespace: keycloak - data: - password: {{ .Values.otomi.adminPassword | b64enc }} - username: {{ .Values.apps.keycloak.adminUsername | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: keycloak-initial-admin + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + username: {{ $k.adminUsername }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-secrets + property: adminPassword {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey {{- end }} diff --git a/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl b/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl index afdb5dd8dd..7ec645c839 100644 --- a/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl +++ b/values/kubeflow-pipelines/kubeflow-pipelines-raw.gotmpl @@ -1,29 +1,60 @@ {{- $v := .Values }} -{{- $kfp := index $v.apps "kubeflow-pipelines" }} {{- $obj := $v.obj.provider }} {{- $httpRoute := tpl (readFile "../../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} {{- $hostname := print "kubeflow-pipelines." $v.cluster.domainSuffix }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - labels: - app: kubeflow-pipelines name: kfp-mysql-secret - data: - password: "{{ $kfp.rootPassword | b64enc }}" - username: "{{ "root" | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: kfp-mysql-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: root + password: '{{ "{{ .rootPassword | toString }}" }}' + data: + - secretKey: rootPassword + remoteRef: + key: kubeflow-pipelines-secrets + property: rootPassword + {{- if eq $obj.type "linode" }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: mlpipeline-obj-artifact + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: mlpipeline-obj-artifact + creationPolicy: Owner + template: + type: Opaque + data: + accesskey: {{ $obj.linode.accessKeyId }} + secretkey: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey + {{- else }} - apiVersion: v1 kind: Secret metadata: - labels: - app: kubeflow-pipelines name: mlpipeline-obj-artifact - data: - {{- if eq $obj.type "linode" }} - accesskey: "{{ $obj.linode.accessKeyId | b64enc }}" - secretkey: "{{ $obj.linode.secretAccessKey | b64enc }}" - {{- end }} + data: {} + {{- end }} - apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/values/loki/loki-raw.gotmpl b/values/loki/loki-raw.gotmpl index a252ef3dce..bbebacc435 100644 --- a/values/loki/loki-raw.gotmpl +++ b/values/loki/loki-raw.gotmpl @@ -4,24 +4,69 @@ {{- $obj := $v.obj.provider }} {{- if $v.otomi.isMultitenant }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: reverse-proxy-auth-config - data: - authn.yaml: {{ tpl (readFile "auth-config.gotmpl") (dict "adminPassword" $l.adminPassword "teams" $v.teamConfig) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: reverse-proxy-auth-config + creationPolicy: Owner + template: + type: Opaque + data: + authn.yaml: | + {{ "{{ " }}$adminPassword := .adminPassword | toString{{ " }}" }} + users: + - username: otomi-admin + password: {{ "\"{{ $adminPassword }}\"" }} + orgid: admins + {{- range $id, $team := $v.teamConfig }} + - username: {{ $id }} + password: {{ printf "\"{{ .team_%s_password | toString }}\"" $id }} + orgid: {{ $id }} + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword + {{- range $id, $team := $v.teamConfig }} + - secretKey: team_{{ $id }}_password + remoteRef: + key: team-{{ $id }}-settings-secrets + property: settings_password + {{- end }} {{- if eq $obj.type "linode" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: loki-s3-linode-credentials - type: Opaque - data: - AWS_ACCESS_KEY_ID: "{{ $obj.linode.accessKeyId | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: loki-s3-linode-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: {{ $obj.linode.accessKeyId }} + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-secrets + property: provider_linode_secretAccessKey {{- end }} {{- end }} diff --git a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl index 77e6be72dc..6cf61ff40a 100644 --- a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl @@ -7,15 +7,54 @@ {{- $ingress := $v.ingress.platformClass }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: oauth2-proxy-redis-password + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-redis-password + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: oauth2-proxy-redis-secrets + property: password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: oauth2-proxy-client-access - type: Opaque - data: - client-id: {{ $k.idp.clientID | b64enc }} - client-secret: {{ $k.idp.clientSecret | b64enc }} - cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-client-access + creationPolicy: Owner + template: + type: Opaque + data: + client-id: {{ $k.idp.clientID }} + client-secret: '{{ "{{ .clientSecret | toString }}" }}' + cookie-secret: '{{ "{{ .cookieSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - secretKey: cookieSecret + remoteRef: + key: oauth2-proxy-secrets + property: config_cookieSecret {{ if $v.apps | dig "ingress-nginx-platform" "enabled" false }} - apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/values/oauth2-proxy/oauth2-proxy.gotmpl b/values/oauth2-proxy/oauth2-proxy.gotmpl index 96807e4283..f573475129 100644 --- a/values/oauth2-proxy/oauth2-proxy.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy.gotmpl @@ -105,7 +105,8 @@ sessionStorage: standalone: connectionUrl: "redis://oauth2-proxy-redis-ha-haproxy.istio-system.svc.cluster.local:6379" {{- end }} - password: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password + passwordKey: password redis-ha: global: @@ -115,7 +116,7 @@ redis-ha: repository: "{{- $v.otomi.linodeLkeImageRepository }}/docker/redis" {{- end }} enabled: true - redisPassword: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password replicas: {{ $r.replicas }} redis: resources: {{- $r.resources.master | toYaml | nindent 6 }} diff --git a/values/otomi-api/otomi-api-raw.gotmpl b/values/otomi-api/otomi-api-raw.gotmpl new file mode 100644 index 0000000000..a3169981dc --- /dev/null +++ b/values/otomi-api/otomi-api-raw.gotmpl @@ -0,0 +1,24 @@ +{{- $v := .Values }} +resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: otomi-api-git-credentials + namespace: otomi + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-api-git-credentials + creationPolicy: Owner + template: + type: Opaque + data: + GIT_PASSWORD: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-secrets + property: git_password diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index 9e55fd1635..ae8fe79618 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -2,11 +2,9 @@ {{- $c := $v.cluster }} {{- $o := $v.apps | get "otomi-api" }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $sops := $v | get "kms.sops" dict }} {{- $giteaValuesPublicUrl := printf "https://gitea.%s/otomi/values" $v.cluster.domainSuffix }} {{- $git := $v.otomi.git }} {{- $defaultPlatformAdminEmail := printf "platform-admin@%s" $v.cluster.domainSuffix }} -{{- $sopsEnv := tpl (readFile "../../helmfile.d/snippets/sops-env.gotmpl") $sops }} {{- $httpRoute := tpl (readFile "../../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} {{- $version := $v.versions | get "api" }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} @@ -39,8 +37,8 @@ tools: secrets: GIT_USER: {{ $git.username | quote }} GIT_EMAIL: {{ $git.email | quote }} - GIT_PASSWORD: {{ $git.password | quote }} - {{- $sopsEnv | nindent 2 }} +gitConfig: + existingSecret: otomi-api-git-credentials env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }} diff --git a/values/prometheus-msteams/prometheus-msteams-raw.gotmpl b/values/prometheus-msteams/prometheus-msteams-raw.gotmpl new file mode 100644 index 0000000000..e105da2e33 --- /dev/null +++ b/values/prometheus-msteams/prometheus-msteams-raw.gotmpl @@ -0,0 +1,30 @@ +{{- $v := .Values }} +resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: prometheus-msteams-connectors + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: prometheus-msteams-connectors + creationPolicy: Owner + template: + type: Opaque + data: + connectors.yaml: | + connectors: + - high_priority_channel: {{ "{{ .highPrio | toString }}" }} + - low_priority_channel: {{ "{{ .lowPrio | toString }}" }} + data: + - secretKey: highPrio + remoteRef: + key: alerts-secrets + property: msteams_highPrio + - secretKey: lowPrio + remoteRef: + key: alerts-secrets + property: msteams_lowPrio diff --git a/values/prometheus-msteams/prometheus-msteams.gotmpl b/values/prometheus-msteams/prometheus-msteams.gotmpl index d766c25c40..d2158b3933 100644 --- a/values/prometheus-msteams/prometheus-msteams.gotmpl +++ b/values/prometheus-msteams/prometheus-msteams.gotmpl @@ -10,9 +10,7 @@ image: pullSecrets: - otomi-pullsecret-global {{- end }} -connectors: -- high_priority_channel: {{ $v.alerts.msteams.highPrio }} -- low_priority_channel: {{ $v.alerts.msteams.lowPrio }} +connectors: [] metrics: serviceMonitor: diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index fff4e04c3b..0805b2c827 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -1,17 +1,147 @@ {{- $v := .Values }} {{- $p := $v.apps | get "prometheus" }} +{{- $a := $v.apps | get "alertmanager" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} -{{- if or ($p | get "remoteWrite.rwConfig.basicAuth.enabled" false) }} +{{- $receivers := $v | get "alerts.receivers" (list "slack") }} +{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} +{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} +{{- $alertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-admin-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: otomi-admin + admin-password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: otomi-secrets + property: adminPassword + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $k.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword {{- if $p | get "remoteWrite.rwConfig.basicAuth.enabled" false }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: prometheus name: prometheus-remote-write-basic-auth - data: - username: {{ $p.remoteWrite.rwConfig.basicAuth.username | b64enc }} - password: {{ $p.remoteWrite.rwConfig.basicAuth.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: prometheus-remote-write-basic-auth + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $p.remoteWrite.rwConfig.basicAuth.username }} + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: prometheus-secrets + property: remoteWrite_rwConfig_basicAuth_password + {{- end }} + {{- if $a.enabled }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-platform-config + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-platform-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $alertmanagerConfig | nindent 14 }} + data: + {{- if has "slack" $receivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $receivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- if has "opsgenie" $receivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} {{- end }} -{{- end }} diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index dbf5082bc0..8c37b3531a 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -12,9 +12,7 @@ {{- $alertmanagerDomain := printf "alertmanager.%s" $domain }} {{- $prometheusDomain := printf "prometheus.%s" $domain }} {{- $grafanaDomain := printf "grafana.%s" $domain }} -{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} -{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} -{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" ($k | get "idp")) | toString }} +{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" (dict "clientID" ($k | get "idp.clientID" "otomi"))) | toString }} {{- $httpRoute := tpl (readFile "../../helmfile.d/snippets/routes.gotmpl") $v | fromYaml }} {{- $hasServices := false }} {{- range $teamId, $team := $v.teamConfig }} @@ -236,7 +234,8 @@ alertmanager: priorityClassName: otomi-critical resources: {{- $a.resources.alertmanager | toYaml | nindent 6 }} externalUrl: https://{{ $alertmanagerDomain }} - config: {{- tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 4 }} + useExistingSecret: true + configSecret: alertmanager-platform-config route: main: enabled: true @@ -308,10 +307,26 @@ grafana: basicAuth: true basicAuthUser: otomi-admin secureJsonData: - basicAuthPassword: {{ $v.apps.loki.adminPassword | quote }} + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} {{- end }} {{- end }} - adminPassword: {{ $g | get "adminPassword" $v.otomi.adminPassword | quote }} + admin: + existingSecret: grafana-admin-secret + userKey: admin-user + passwordKey: admin-password + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: + secretKeyRef: + name: grafana-oidc-secret + key: client_id + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc-secret + key: client_secret + GF_LOKI_BASIC_AUTH_PASSWORD: + secretKeyRef: + name: grafana-loki-datasource-secret + key: password grafana.ini: {{- $grafanaIni | nindent 4 }} server: root_url: https://{{ $grafanaDomain }} diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl new file mode 100644 index 0000000000..1b2f66c1b8 --- /dev/null +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -0,0 +1,125 @@ +{{- $v := .Values }} +{{- $teamId := .Release.Labels.team }} +{{- $tc := $v.teamConfig }} +{{- $teamSettings := (index $tc $teamId).settings }} +{{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} +{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} +{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} +{{- $teamAlertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} + +resources: +{{- if $teamSettings | get "managedMonitoring.grafana" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: team-{{ $teamId }}-grafana-admin + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: team-{{ $teamId }}-grafana-admin + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: {{ $teamId }} + admin-password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: team-{{ $teamId }}-settings-secrets + property: settings_password +{{- end }} +{{- if $teamSettings | get "managedMonitoring.grafana" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $v.apps.keycloak.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword +{{- end }} +{{- if $teamSettings | get "managedMonitoring.alertmanager" false }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-team-{{ $teamId }}-config + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-team-{{ $teamId }}-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $teamAlertmanagerConfig | nindent 12 }} + data: + {{- if has "slack" $teamReceivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $teamReceivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} +{{- end }} diff --git a/versions.yaml b/versions.yaml index f3c2a59e11..ebf3ce350b 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,6 +1,6 @@ -api: main -console: main -consoleLogin: main -tasks: main +api: APL-523 +console: APL-523 +consoleLogin: APL-523 +tasks: APL-523 tools: main aplCharts: main