diff --git a/valkey/templates/_helpers.tpl b/valkey/templates/_helpers.tpl index 52684297..0dd2c16e 100644 --- a/valkey/templates/_helpers.tpl +++ b/valkey/templates/_helpers.tpl @@ -196,8 +196,11 @@ Validate sentinel configuration {{- if not .Values.replica.enabled }} {{- fail "Sentinel mode requires replication to be enabled. Please set replica.enabled=true along with sentinel.enabled=true" }} {{- end }} - {{- if lt (add (int .Values.replica.replicas) 1) 3 }} - {{- fail "Sentinel mode requires at least 3 Valkey pods (replicas: 2) for a stable quorum." }} + {{- if lt (int .Values.replica.sentinel.replicas) 3 }} + {{- fail "Sentinel mode requires at least 3 pods for a stable quorum." }} + {{- end }} + {{- if gt (int .Values.replica.sentinel.quorum) (int .Values.replica.sentinel.replicas) }} + {{- fail (printf "Sentinel quorum (%d) cannot be greater than sentinels count (%d)." (int .Values.replica.sentinel.quorum) (int .Values.replica.sentinel.replicas)) }} {{- end }} {{- if and .Values.auth.enabled (not (hasKey .Values.auth.aclUsers .Values.replica.replicationUser)) }} {{- fail (printf "Sentinel with auth requires replication user '%s' to be defined in auth.aclUsers" .Values.replica.replicationUser) }} @@ -219,3 +222,11 @@ Sentinel headless service name {{ include "valkey.fullname" . }}-sentinel-headless {{- end -}} +{{/* +Validate haproxy is used in replica mode +*/}} +{{- define "valkey.validateHaproxyRequirements" -}} +{{- if and .Values.haproxy.enabled (or (not .Values.replica.enabled) (not .Values.replica.sentinel.enabled)) }} + {{- fail "Haproxy requires replica mode and sentinel to handle incompatible sentinel clients usescases." }} +{{- end }} +{{- end -}} diff --git a/valkey/templates/deploy_valkey.yaml b/valkey/templates/deploy_valkey.yaml index da7cd712..215c30c4 100644 --- a/valkey/templates/deploy_valkey.yaml +++ b/valkey/templates/deploy_valkey.yaml @@ -3,6 +3,7 @@ {{- $storage := .Values.dataStorage }} {{- $createPVC := and $storage.enabled (not (empty $storage.requestedSize)) (empty $storage.persistentVolumeClaimName) }} {{- include "valkey.validateAuthConfig" . }} +{{- include "valkey.validateHaproxyRequirements" . }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/valkey/templates/haproxy-configmap.yaml b/valkey/templates/haproxy-configmap.yaml index af67c765..b85140a3 100644 --- a/valkey/templates/haproxy-configmap.yaml +++ b/valkey/templates/haproxy-configmap.yaml @@ -19,19 +19,15 @@ data: timeout server {{ .Values.haproxy.config.timeout.server }} retries 3 - frontend valkey_frontend_write - bind *:{{ .Values.haproxy.service.port | default 6379 }} + {{- range .Values.haproxy.service.ports }} + frontend valkey_frontend_{{ .name }} + bind *:{{ .port }} mode tcp option tcplog - default_backend valkey_backend_master - - frontend valkey_frontend_read - bind *:{{ .Values.haproxy.service.readPort | default 6380 }} - mode tcp - option tcplog - default_backend valkey_backend_read + default_backend valkey_backend_{{ .name }} + {{- end }} - backend valkey_backend_master + backend valkey_backend_{{ (index .Values.haproxy.service.ports 0).name }} mode tcp # no-check: sentinel-watcher solely controls which server is active via # the runtime socket (set server addr + enable/disable server). @@ -41,7 +37,7 @@ data: server valkey-{{ $i }} 127.0.0.1:6379 check-send-proxy disabled {{- end }} - backend valkey_backend_read + backend valkey_backend_{{ (index .Values.haproxy.service.ports 1).name }} mode tcp option tcp-check @@ -72,7 +68,7 @@ data: # Sentinel watcher: polls Sentinel for master changes and updates HAProxy # via the runtime socket. # - # KEY DESIGN: All valkey_backend_master servers are configured as + # KEY DESIGN: All valkey_backend_{{ (index .Values.haproxy.service.ports 0).name }} servers are configured as # `no-check disabled` in haproxy.cfg. This watcher is the SOLE controller # of which server is active. It resolves the master hostname to a Pod IP # and uses `set server addr` so HAProxy connects directly to the IP, @@ -85,7 +81,8 @@ data: POLL_INTERVAL="{{ .Values.haproxy.config.checkInterval | default 2 }}" VALKEY_PORT="{{ .Values.service.port }}" HAPROXY_SOCKET="/var/run/haproxy/admin.sock" - BACKEND="valkey_backend_master" + BACKEND="valkey_backend_{{ (index .Values.haproxy.service.ports 0).name }}" + TOTAL_SENTINELS="{{ .Values.replica.sentinel.replicas }}" TOTAL_SERVERS="{{ add (int .Values.replica.replicas) 1 }}" # Logging to stderr prevents polluting stdout @@ -148,8 +145,8 @@ data: # 1. Ask Sentinels who the master is i=0 - while [ "${i}" -lt "${TOTAL_SERVERS}" ]; do - S_HOST="{{ include "valkey.fullname" . }}-${i}.{{ include "valkey.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + while [ "${i}" -lt "${TOTAL_SENTINELS}" ]; do + S_HOST="{{ include "valkey.fullname" . }}-sentinel-${i}.{{ include "valkey.sentinel.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" RESP=$(sentinel_cmd "${S_HOST}" sentinel get-master-addr-by-name "${MASTER_SET}" 2>/dev/null) || { i=$((i + 1)) diff --git a/valkey/templates/haproxy-deployment.yaml b/valkey/templates/haproxy-deployment.yaml index 51fa5177..8692b85a 100644 --- a/valkey/templates/haproxy-deployment.yaml +++ b/valkey/templates/haproxy-deployment.yaml @@ -1,4 +1,5 @@ {{- if .Values.haproxy.enabled }} +{{- include "valkey.validateHaproxyRequirements" . }} apiVersion: apps/v1 kind: Deployment metadata: @@ -47,12 +48,11 @@ spec: {{- end }} imagePullPolicy: {{ .Values.haproxy.image.pullPolicy }} ports: - - name: valkey-write - containerPort: 6379 - protocol: TCP - - name: valkey-read - containerPort: 6380 + {{- range .Values.haproxy.service.ports }} + - name: {{ .name }} + containerPort: {{ .port }} protocol: TCP + {{- end }} volumeMounts: - name: haproxy-config mountPath: /usr/local/etc/haproxy/haproxy.cfg diff --git a/valkey/templates/haproxy-service.yaml b/valkey/templates/haproxy-service.yaml index b89d83b7..1e24d568 100644 --- a/valkey/templates/haproxy-service.yaml +++ b/valkey/templates/haproxy-service.yaml @@ -13,14 +13,19 @@ metadata: spec: type: {{ .Values.haproxy.service.type }} ports: - - port: {{ .Values.haproxy.service.port | default 6379 }} - targetPort: {{ .Values.haproxy.service.port | default 6379 }} + {{- $serviceType := .Values.haproxy.service.type }} + {{- range .Values.haproxy.service.ports }} + - port: {{ .port }} + targetPort: {{ .port }} protocol: TCP - name: valkey-write - - port: {{ .Values.haproxy.service.readPort | default 6380 }} - targetPort: {{ .Values.haproxy.service.readPort | default 6380 }} - protocol: TCP - name: valkey-read + name: {{ .name }} + {{- if and (eq $serviceType "NodePort") .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + {{- if .appProtocol }} + appProtocol: {{ .appProtocol }} + {{- end }} + {{- end }} selector: {{- include "valkey.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: haproxy diff --git a/valkey/templates/init_config.yaml b/valkey/templates/init_config.yaml index cb712c5b..f1384d4f 100644 --- a/valkey/templates/init_config.yaml +++ b/valkey/templates/init_config.yaml @@ -11,6 +11,7 @@ data: # Default config paths VALKEY_CONFIG=${VALKEY_CONFIG_PATH:-/data/conf/valkey.conf} + BOOTSTRAP_FILE="/data/conf/.bootstrap" LOGFILE="/data/init.log" DATA_DIR="/data/conf" @@ -175,7 +176,10 @@ data: # 1. Determine Initial Topology or Restored State REPLICA_TARGET="" if [ "${SENTINEL_MODE:-false}" = "true" ]; then - if [ -n "$SAVED_REPLICA_LINE" ]; then + if [ -f "$BOOTSTRAP_FILE" ] && [ -z "$SAVED_REPLICA_LINE" ] ; then + log "Sentinel mode: Restoring master state from previous run" + REPLICA_TARGET="" + elif [ -n "$SAVED_REPLICA_LINE" ]; then # CRITICAL: Restore topology written by Sentinel (CONFIG REWRITE) log "Sentinel Mode: Restoring replication state from previous run: $SAVED_REPLICA_LINE" REPLICA_TARGET="$SAVED_REPLICA_LINE" @@ -269,3 +273,4 @@ data: log "Appending files in /extravalkeyconfigs/" cat /extravalkeyconfigs/* >>"$VALKEY_CONFIG" fi + touch $BOOTSTRAP_FILE diff --git a/valkey/templates/sentinel-configmap.yaml b/valkey/templates/sentinel-configmap.yaml index a1e1474a..c66347fe 100644 --- a/valkey/templates/sentinel-configmap.yaml +++ b/valkey/templates/sentinel-configmap.yaml @@ -19,17 +19,46 @@ data: FAILOVER_TIMEOUT="{{ .Values.replica.sentinel.failoverTimeout }}" PARALLEL_SYNCS="{{ .Values.replica.sentinel.parallelSyncs }}" - # Master is always pod-0 of the valkey statefulset initially - MASTER_HOST="{{ include "valkey.fullname" . }}-0.{{ include "valkey.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" - MASTER_PORT="{{ .Values.service.port }}" - - SENTINEL_CONF="/data/sentinel.conf" - # Logging function (outputs to stderr to avoid polluting stdout command substitutions) log() { echo "$(date) $1" >&2 } + has_master() { + for i in $(seq 0 "{{ (int .Values.replica.replicas) }}"); do + valkey-cli -h {{ include "valkey.fullname" . }}-$i.{{ include "valkey.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} INFO replication 2>/dev/null \ + | grep -q "^role:master" && return 0 + done + return 1 + } + + # Wait until a master is UP + until has_master; do + log "Master not ready" + sleep 5 + done + + # Discover who is the current master + MASTER_HOST="" + for i in $(seq 0 "{{ (int .Values.replica.replicas) }}"); do + HOST="{{ include "valkey.fullname" . }}-$i.{{ include "valkey.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + + ROLE=$(redis-cli -h "$HOST" INFO replication 2>/dev/null \ + | awk -F: '/^role:/ {print $2}' \ + | tr -d '\r') + + if [ "$ROLE" = "master" ]; then + MASTER_HOST="$HOST" + break + fi + + i=$((i + 1)) + done + MASTER_PORT="{{ .Values.service.port }}" + log "Found existing master $MASTER_HOST" + + SENTINEL_CONF="/sentinel-data/sentinel.conf" + log "Initializing Sentinel configuration..." {{- if .Values.auth.enabled }} @@ -78,7 +107,7 @@ data: # Announce IP for discovery in Kubernetes sentinel resolve-hostnames yes sentinel announce-hostnames yes - sentinel announce-ip ${POD_NAME}.{{ include "valkey.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + sentinel announce-ip ${POD_NAME}.{{ include "valkey.sentinel.headlessServiceName" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} sentinel announce-port ${SENTINEL_PORT} # Monitor the master diff --git a/valkey/templates/sentinel-statefulset.yaml b/valkey/templates/sentinel-statefulset.yaml new file mode 100644 index 00000000..40baf866 --- /dev/null +++ b/valkey/templates/sentinel-statefulset.yaml @@ -0,0 +1,195 @@ +{{- if and .Values.replica.enabled .Values.replica.sentinel.enabled }} +{{- include "valkey.validateSentinelConfig" . }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "valkey.fullname" . }}-sentinel + labels: + {{- include "valkey.labels" . | nindent 4 }} + app.kubernetes.io/component: sentinel +spec: + serviceName: {{ include "valkey.fullname" . }}-sentinel-headless + replicas: {{ .Values.replica.sentinel.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "valkey.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: sentinel + {{- if .Values.replica.sentinel.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: sentinel-data + spec: + accessModes: {{ toYaml .Values.replica.sentinel.persistence.accessModes | nindent 8 }} + {{- if .Values.replica.sentinel.persistence.storageClass }} + storageClassName: {{ .Values.replica.sentinel.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.replica.sentinel.persistence.size | quote }} + {{- end }} + template: + metadata: + labels: + {{- include "valkey.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: sentinel + {{- with .Values.commonLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/sentinel-config: {{ include (print $.Template.BasePath "/sentinel-configmap.yaml") . | sha256sum | trunc 32 | quote }} + spec: + {{- (include "valkey.imagePullSecrets" .) | nindent 6 }} + automountServiceAccountToken: {{ .Values.serviceAccount.automount }} + serviceAccountName: {{ include "valkey.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: {{ include "valkey.fullname" . }}-sentinel-init + image: {{ include "valkey.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: [ "/sentinel-scripts/sentinel-init.sh" ] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + {{- if .Values.replica.sentinel.persistence.enabled }} + - name: sentinel-data + mountPath: /sentinel-data + {{- else }} + - name: sentinel-data + mountPath: /sentinel-data + {{- end }} + - name: sentinel-scripts + mountPath: /sentinel-scripts + {{- if .Values.auth.enabled }} + {{- if .Values.auth.usersExistingSecret }} + - name: valkey-users-secret + mountPath: /valkey-users-secret + readOnly: true + {{- end }} + {{- if or (include "valkey.hasInlinePasswords" . | eq "true") .Values.auth.aclConfig }} + - name: valkey-auth-secret + mountPath: /valkey-auth-secret + readOnly: true + {{- end }} + {{- end }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: sentinel + image: {{ include "valkey.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ "valkey-sentinel" ] + workingDir: /sentinel-data + args: [ "/sentinel-data/sentinel.conf" ] + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + ports: + - name: sentinel + containerPort: {{ .Values.replica.sentinel.port }} + protocol: TCP + startupProbe: + exec: + {{- if .Values.tls.enabled }} + command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} --cacert /tls/{{ .Values.tls.caPublicKey }} --tls ping" ] + {{- else }} + command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} ping" ] + {{- end }} + livenessProbe: + exec: + {{- if .Values.tls.enabled }} + command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} --cacert /tls/{{ .Values.tls.caPublicKey }} --tls ping" ] + {{- else }} + command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} ping" ] + {{- end }} + resources: + {{- toYaml .Values.replica.sentinel.resources | nindent 12 }} + volumeMounts: + {{- if .Values.replica.sentinel.persistence.enabled }} + - name: sentinel-data + mountPath: /sentinel-data + {{- else }} + - name: sentinel-data + mountPath: /sentinel-data + {{- end }} + {{- if .Values.tls.enabled }} + - name: {{ include "valkey.fullname" . }}-tls + mountPath: /tls + {{- end }} + volumes: + - name: sentinel-scripts + configMap: + name: {{ include "valkey.fullname" . }}-sentinel-scripts + defaultMode: 0555 + {{- if not .Values.replica.sentinel.persistence.enabled }} + - name: sentinel-data + emptyDir: {} + {{- end }} + {{- if .Values.tls.enabled }} + - name: {{ include "valkey.fullname" . }}-tls + secret: + secretName: {{ required "An existing secret is required to enable TLS" .Values.tls.existingSecret }} + defaultMode: 0400 + {{- end }} + {{- if .Values.auth.enabled }} + {{- if .Values.auth.usersExistingSecret }} + - name: valkey-users-secret + secret: + secretName: {{ .Values.auth.usersExistingSecret }} + defaultMode: 0400 + {{- end }} + {{- if or (include "valkey.hasInlinePasswords" . | eq "true") .Values.auth.aclConfig }} + - name: valkey-auth-secret + secret: + secretName: {{ include "valkey.fullname" . }}-auth + defaultMode: 0400 + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/valkey/templates/service-headless.yaml b/valkey/templates/service-headless.yaml index 073ce4f0..733ca683 100644 --- a/valkey/templates/service-headless.yaml +++ b/valkey/templates/service-headless.yaml @@ -15,12 +15,6 @@ spec: port: {{ .Values.service.port }} targetPort: tcp protocol: TCP - {{- if .Values.replica.sentinel.enabled }} - - name: sentinel - port: {{ .Values.replica.sentinel.port }} - targetPort: sentinel - protocol: TCP - {{- end }} selector: {{- include "valkey.selectorLabels" . | nindent 4 }} {{- end }} diff --git a/valkey/templates/service-read.yaml b/valkey/templates/service-read.yaml index 49ec54e7..96b9da0b 100644 --- a/valkey/templates/service-read.yaml +++ b/valkey/templates/service-read.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.replica.enabled .Values.replica.service.enabled }} +{{- if and .Values.replica.enabled .Values.replica.service.enabled (not .Values.replica.sentinel.enabled) }} apiVersion: v1 kind: Service metadata: diff --git a/valkey/templates/service.yaml b/valkey/templates/service.yaml index 1e786826..a8efaceb 100644 --- a/valkey/templates/service.yaml +++ b/valkey/templates/service.yaml @@ -4,7 +4,9 @@ metadata: name: {{ include "valkey.fullname" . }} labels: {{- include "valkey.labels" . | nindent 4 }} + {{- if and .Values.replica.enabled (not .Values.replica.sentinel.enabled) }} app.kubernetes.io/component: primary + {{- end }} {{- with .Values.service.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -33,6 +35,6 @@ spec: {{- end }} selector: {{- include "valkey.selectorLabels" . | nindent 4 }} - {{- if .Values.replica.enabled }} + {{- if and .Values.replica.enabled (not .Values.replica.sentinel.enabled) }} statefulset.kubernetes.io/pod-name: {{ include "valkey.fullname" . }}-0 {{- end }} diff --git a/valkey/templates/statefulset.yaml b/valkey/templates/statefulset.yaml index a9632537..bc58bda6 100644 --- a/valkey/templates/statefulset.yaml +++ b/valkey/templates/statefulset.yaml @@ -11,7 +11,7 @@ metadata: spec: serviceName: {{ include "valkey.fullname" . }}-headless replicas: {{ add (int .Values.replica.replicas) 1 }} - podManagementPolicy: Parallel + podManagementPolicy: OrderedReady {{- if .Values.replica.persistentVolumeClaimRetentionPolicy }} persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.replica.persistentVolumeClaimRetentionPolicy | nindent 4 }} @@ -30,26 +30,11 @@ spec: resources: requests: storage: {{ .Values.replica.persistence.size | quote }} - {{- if and .Values.replica.sentinel.enabled .Values.replica.sentinel.persistence.enabled }} - - metadata: - name: sentinel-data - spec: - accessModes: {{ toYaml .Values.replica.sentinel.persistence.accessModes | nindent 8 }} - {{- if .Values.replica.sentinel.persistence.storageClass }} - storageClassName: {{ .Values.replica.sentinel.persistence.storageClass | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.replica.sentinel.persistence.size | quote }} - {{- end }} template: metadata: labels: {{- include "valkey.selectorLabels" . | nindent 8 }} - {{- if .Values.replica.sentinel.enabled }} - app.kubernetes.io/component: sentinel - {{- end }} {{- with .Values.commonLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -68,9 +53,6 @@ spec: {{- (include "valkey.imagePullSecrets" .) | nindent 6 }} automountServiceAccountToken: {{ .Values.serviceAccount.automount }} serviceAccountName: {{ include "valkey.serviceAccountName" . }} - {{- if .Values.replica.sentinel.enabled }} - shareProcessNamespace: true - {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName | quote }} {{- end }} @@ -129,51 +111,6 @@ spec: {{- with .Values.extraInitContainers }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.replica.sentinel.enabled }} - - name: {{ include "valkey.fullname" . }}-sentinel-init - image: {{ include "valkey.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- with .Values.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: [ "/sentinel-scripts/sentinel-init.sh" ] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - {{- if .Values.replica.sentinel.persistence.enabled }} - - name: sentinel-data - mountPath: /data - {{- else }} - - name: sentinel-data - mountPath: /data - {{- end }} - - name: sentinel-scripts - mountPath: /sentinel-scripts - {{- if .Values.auth.enabled }} - {{- if .Values.auth.usersExistingSecret }} - - name: valkey-users-secret - mountPath: /valkey-users-secret - readOnly: true - {{- end }} - {{- if or (include "valkey.hasInlinePasswords" . | eq "true") .Values.auth.aclConfig }} - - name: valkey-auth-secret - mountPath: /valkey-auth-secret - readOnly: true - {{- end }} - {{- end }} - {{- with .Values.initResources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} containers: - name: {{ include "valkey.fullname" . }} image: {{ include "valkey.image" . }} @@ -235,56 +172,6 @@ spec: {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.replica.sentinel.enabled }} - - name: sentinel - image: {{ include "valkey.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: [ "valkey-sentinel" ] - args: [ "/sentinel-data/sentinel.conf" ] - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - {{- range $key, $val := .Values.env }} - - name: {{ $key }} - value: "{{ $val }}" - {{- end }} - ports: - - name: sentinel - containerPort: {{ .Values.replica.sentinel.port }} - protocol: TCP - startupProbe: - exec: - {{- if .Values.tls.enabled }} - command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} --cacert /tls/{{ .Values.tls.caPublicKey }} --tls ping" ] - {{- else }} - command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} ping" ] - {{- end }} - livenessProbe: - exec: - {{- if .Values.tls.enabled }} - command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} --cacert /tls/{{ .Values.tls.caPublicKey }} --tls ping" ] - {{- else }} - command: [ "sh", "-c", "valkey-cli -p {{ .Values.replica.sentinel.port }} ping" ] - {{- end }} - resources: - {{- toYaml .Values.replica.sentinel.resources | nindent 12 }} - volumeMounts: - {{- if .Values.replica.sentinel.persistence.enabled }} - - name: sentinel-data - mountPath: /sentinel-data - {{- else }} - - name: sentinel-data - mountPath: /sentinel-data - {{- end }} - {{- if .Values.tls.enabled }} - - name: {{ include "valkey.fullname" . }}-tls - mountPath: /tls - {{- end }} - {{- end }} {{- if .Values.metrics.enabled }} - name: metrics image: {{ include "valkey.metrics.exporter.image" . }} @@ -331,16 +218,6 @@ spec: {{- end }} {{- end }} volumes: - {{- if .Values.replica.sentinel.enabled }} - - name: sentinel-scripts - configMap: - name: {{ include "valkey.fullname" . }}-sentinel-scripts - defaultMode: 0555 - {{- if not .Values.replica.sentinel.persistence.enabled }} - - name: sentinel-data - emptyDir: {} - {{- end }} - {{- end }} - name: scripts configMap: name: {{ include "valkey.fullname" . }}-init-scripts @@ -414,4 +291,4 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/valkey/tests/haproxy_test.yaml b/valkey/tests/haproxy_test.yaml new file mode 100644 index 00000000..80f25240 --- /dev/null +++ b/valkey/tests/haproxy_test.yaml @@ -0,0 +1,246 @@ +suite: sentinel configuration +templates: + - templates/haproxy-deployment.yaml + - templates/haproxy-service.yaml + - templates/haproxy-configmap.yaml +tests: + - it: should fail when haproxy is enabled but sentinel is disabled + set: + haproxy.enabled: true + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: false + template: templates/haproxy-deployment.yaml + asserts: + - failedTemplate: + errorPattern: "Haproxy requires replica mode and sentinel to handle incompatible sentinel clients usescases." + + - it: should not create haproxy Deployment when replica is disabled (default) + set: + haproxy.enabled: true + replica.enabled: false + template: templates/haproxy-deployment.yaml + asserts: + - failedTemplate: + errorPattern: "Haproxy requires replica mode and sentinel to handle incompatible sentinel clients usescases." + + - it: should create haproxy Deployment when haproxy is enabled + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.replicas: 3 + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: metadata.name + value: RELEASE-NAME-valkey-haproxy + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 6379 + - equal: + path: spec.template.spec.containers[0].ports[1].containerPort + value: 6380 + + - it: should create haproxy Service when haproxy is enabled + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + template: templates/haproxy-service.yaml + asserts: + - isKind: + of: Service + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: haproxy + - equal: + path: spec.selector["app.kubernetes.io/component"] + value: haproxy + - equal: + path: metadata.name + value: RELEASE-NAME-valkey-haproxy + - equal: + path: spec.ports[0].port + value: 6379 + - equal: + path: spec.ports[1].port + value: 6380 + + - it: should create haproxy ConfigMap when haproxy is enabled + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + template: templates/haproxy-configmap.yaml + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: RELEASE-NAME-valkey-haproxy + - exists: + path: data["haproxy.cfg"] + - exists: + path: data["sentinel-watcher.sh"] + + - it: should use custom haproxy replicas count + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.replicas: 4 + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 4 + + - it: should use custom haproxy port + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.service.ports: + - name: valkey-write + port: 6389 + targetPort: 6389 + - name: valkey-read + port: 6390 + targetPort: 6390 + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 6389 + - equal: + path: spec.template.spec.containers[0].ports[1].containerPort + value: 6390 + + - it: should use emptyDir + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - contains: + path: spec.template.spec.volumes + content: + name: haproxy-socket + emptyDir: {} + + - it: should set correct service type and nodePort + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.service.type: NodePort + haproxy.service.ports: + - name: valkey-write + port: 6389 + targetPort: 6389 + nodePort: 31379 + - name: valkey-read + port: 6390 + targetPort: 6390 + nodePort: 31380 + template: templates/haproxy-service.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: NodePort + - equal: + path: spec.ports[0].nodePort + value: 31379 + - equal: + path: spec.ports[1].nodePort + value: 31380 + + - it: should include haproxy component label + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: haproxy + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/component"] + value: haproxy + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: haproxy + + - it: should apply resources when configured + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 200m + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 256Mi + + - it: should apply resources for sentinelWatcher when configured + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + haproxy.enabled: true + haproxy.sentinelWatcher.resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + template: templates/haproxy-deployment.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[1].resources.requests.cpu + value: 100m + - equal: + path: spec.template.spec.containers[1].resources.requests.memory + value: 128Mi \ No newline at end of file diff --git a/valkey/tests/sentinel_test.yaml b/valkey/tests/sentinel_test.yaml index 7cb27504..dda8bd81 100644 --- a/valkey/tests/sentinel_test.yaml +++ b/valkey/tests/sentinel_test.yaml @@ -32,6 +32,29 @@ tests: - hasDocuments: count: 0 + - it: should fail when sentinel replicas count is too low for a stable quorum + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + replica.sentinel.replicas: 2 + template: templates/sentinel-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: "Sentinel mode requires at least 3 pods for a stable quorum.*" + + - it: should fail when quorum is greater than sentinels count + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + replica.sentinel.replicas: 3 + replica.sentinel.quorum: 4 + template: templates/sentinel-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: "Sentinel quorum (4) cannot be greater than sentinels count (3)." + - it: should create sentinel StatefulSet when both replica and sentinel are enabled set: replica.enabled: true diff --git a/valkey/tests/service_read_test.yaml b/valkey/tests/service_read_test.yaml index 079b59f6..4d43550b 100644 --- a/valkey/tests/service_read_test.yaml +++ b/valkey/tests/service_read_test.yaml @@ -114,3 +114,13 @@ tests: asserts: - hasDocuments: count: 0 + - it: should not create read service when sentinel is enabled + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + replica.sentinel.replicas: 3 + template: templates/service-read.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/valkey/tests/service_test.yaml b/valkey/tests/service_test.yaml index 115c137a..39e4d1ec 100644 --- a/valkey/tests/service_test.yaml +++ b/valkey/tests/service_test.yaml @@ -86,3 +86,19 @@ tests: content: app.kubernetes.io/instance: RELEASE-NAME app.kubernetes.io/name: valkey + - it: should not set the pod-0 selector when sentinel is enabled + set: + replica.enabled: true + replica.persistence.size: "5Gi" + replica.sentinel.enabled: true + replica.sentinel.replicas: 3 + template: templates/service.yaml + asserts: + - isKind: + of: Service + - notExists: + path: metadata.labels["app.kubernetes.io/component"] + value: primary + - notMatchRegex: + path: spec.selector["statefulset.kubernetes.io/pod-name"] + pattern: '-0$' diff --git a/valkey/values.schema.json b/valkey/values.schema.json index f7e49bbd..fe1dbfc8 100644 --- a/valkey/values.schema.json +++ b/valkey/values.schema.json @@ -153,13 +153,10 @@ "type": "object", "properties": { "type": { - "type": "string" - }, - "port": { - "type": "integer" + "type": "string" }, - "annotations": { - "type": "object" + "ports": { + "type": "array" } } }, diff --git a/valkey/values.yaml b/valkey/values.yaml index d9cec4b0..c7829b36 100644 --- a/valkey/values.yaml +++ b/valkey/values.yaml @@ -275,6 +275,9 @@ replica: # Enable Sentinel for high availability and automatic failover enabled: false + # Number of sentinel instances (needs at least 3 replicas for the quorum) + replicas: 3 + # Username for Sentinel to authenticate to the Valkey master/replicas. # If not defined or left empty, the script will fallback to using replicationUser. # WARNING: If the 'default' user is disabled in auth.aclUsers, this MUST be set to a @@ -557,10 +560,27 @@ haproxy: pullPolicy: IfNotPresent replicas: 3 service: + # Service type (ClusterIP, NodePort, LoadBalancer) type: ClusterIP - port: 6379 # Port for write operations - readPort: 6380 # Port for read operations - annotations: {} + ports: + # Port name for read-write opérations + - name: valkey-write + # Port for read-write opérations + port: 6379 + # NodePort value (if service.type is NodePort) + nodePort: 0 + # Port name for read only opérations + - name: valkey-read + # Port for read only opérations + port: 6380 + # NodePort value (if service.type is NodePort) + nodePort: 0 + # Optional annotations for the sentinel service + annotations: {} + # ClusterIP value + clusterIP: "" + # Application protocol + appProtocol: "" resources: {} # limits: # cpu: 100m