From 2e7590282b7a1bde33cb7bd524803e0e9460bc49 Mon Sep 17 00:00:00 2001 From: phil Date: Sun, 28 Jun 2026 15:13:19 +0900 Subject: [PATCH] fix(resources): TLS cluster-bus announce-port = tls-port+10000 (16380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 근본원인: --cluster-announce-bus-port 가 PortClusterBus(16379) 하드코딩, TLS 클러스터(tls-cluster yes)의 실제 bus listen=tls-port+10000(16380) 와 불일치 → 전 노드가 닫힌 16379 로 gossip → fail?/disconnected → cluster_state:fail, slots_ok:0 (keiailab-valkey-prod 23일 고착, 4증거 삼각검증: 소스+config+nc+로그). clusterAnnounceCommand 를 tls-aware 로: bus = tlsEnabled ? PortTLS+10000 : PortClient+10000. 회귀가드: builders_basic_test TLS(16380)/non-TLS(16379) 서브테스트. chart version/appVersion 1.5.1 정합 (release.yml preflight: version==appVersion==tag). 검증: go build/vet/test green. Co-Authored-By: Claude Opus 4.8 Signed-off-by: phil --- charts/valkey-operator/Chart.yaml | 10 ++++---- internal/resources/builders_basic_test.go | 31 +++++++++++++++++++++++ internal/resources/statefulset.go | 17 ++++++++++--- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/charts/valkey-operator/Chart.yaml b/charts/valkey-operator/Chart.yaml index b188c39..58f9139 100644 --- a/charts/valkey-operator/Chart.yaml +++ b/charts/valkey-operator/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: valkey-operator description: A Kubernetes Operator for managing Valkey instances and Clusters (Redis OSS fork, BSD-licensed) type: application -version: 1.4.0 -appVersion: "1.5.0" +version: 1.5.1 +appVersion: "1.5.1" kubeVersion: ">=1.26.0-0" keywords: @@ -121,13 +121,13 @@ annotations: artifacthub.io/containsSecurityUpdates: "true" artifacthub.io/images: | - name: valkey-operator - image: ghcr.io/keiailab/valkey-operator:1.5.0 + image: ghcr.io/keiailab/valkey-operator:1.5.1 - name: valkey image: docker.io/valkey/valkey:9.1.0-alpine3.23 artifacthub.io/prerelease: "false" artifacthub.io/changes: | - - kind: changed - description: Bump chart to appVersion 1.5.0 (operator image 1.5.0) — chart had lagged at 1.3.0 while ghcr image advanced to 1.5.0. + - kind: fixed + description: "TLS cluster: announce cluster-bus port as tls-port+10000 (16380) instead of hardcoded 16379 — fixes inter-node gossip failure (cluster_state:fail / slots_ok:0) on tls-cluster deployments." artifacthub.io/recommendations: | - url: https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack - url: https://artifacthub.io/packages/helm/cert-manager/cert-manager diff --git a/internal/resources/builders_basic_test.go b/internal/resources/builders_basic_test.go index 4ef91af..777b1c8 100644 --- a/internal/resources/builders_basic_test.go +++ b/internal/resources/builders_basic_test.go @@ -406,6 +406,37 @@ func TestBuildStatefulSet(t *testing.T) { t.Error("cluster mode → cluster-bus port (16379) 누락") } }) + t.Run("cluster mode TLS announces bus port = tls-port+10000 (gossip 단절 회귀가드)", func(t *testing.T) { + t.Parallel() + // 회귀 사고: tls-cluster 에서 bus 는 tls-port(6380)+10000=16380 에 listen 하나 + // operator 가 16379(=비-TLS port+10000) 를 announce → peer 가 닫힌 16379 로 + // gossip 시도 → 전 노드 disconnected → cluster_state:fail, slots_ok:0 (23일 고착). + p := makeParams() + p.ClusterMode = true + p.TLSSecretName = "rs-tls" // TLS 활성 + sts := BuildStatefulSet(p) + c := sts.Spec.Template.Spec.Containers[0] + if len(c.Command) != 3 || c.Command[0] != "sh" || c.Command[1] != "-c" { + t.Fatalf("cluster mode 는 sh -c 래핑 command 기대, got %v", c.Command) + } + shCmd := c.Command[2] + if !strings.Contains(shCmd, "--cluster-announce-bus-port 16380") { + t.Errorf("TLS cluster 는 announce-bus-port 16380 기대 (tls-port+10000), got: %s", shCmd) + } + if strings.Contains(shCmd, "--cluster-announce-bus-port 16379") { + t.Errorf("TLS cluster 가 비-TLS bus port 16379 announce — gossip 실패: %s", shCmd) + } + }) + t.Run("cluster mode non-TLS announces bus port = port+10000 (16379)", func(t *testing.T) { + t.Parallel() + p := makeParams() + p.ClusterMode = true // TLS 미설정 → bus = port(6379)+10000 = 16379 + sts := BuildStatefulSet(p) + shCmd := sts.Spec.Template.Spec.Containers[0].Command[2] + if !strings.Contains(shCmd, "--cluster-announce-bus-port 16379") { + t.Errorf("non-TLS cluster 는 announce-bus-port 16379 기대, got: %s", shCmd) + } + }) t.Run("password env via SecretKeyRef", func(t *testing.T) { t.Parallel() sts := BuildStatefulSet(makeParams()) diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index 9431685..15b0daf 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -222,9 +222,10 @@ func BuildStatefulSet(p STSParams) *appsv1.StatefulSet { // 컨테이너 command 에서 $POD_IP 를 셸 확장한다. CLI flag 가 valkey.conf 보다 // 우선하므로 ConfigMap 의 기타 directive 는 그대로 유효. // announce-port=client(6379/tls 6380 무관 — 6379 는 평문/내부 dial 기준), - // announce-bus-port=cluster-bus(16379). + // announce-bus-port=tls-cluster 시 tls-port+10000(16380) / 아니면 port+10000(16379) + // — 실제 bus listen 포트와 일치해야 gossip 성립 (clusterAnnounceCommand 참조). if p.ClusterMode { - containers[0].Command, containers[0].Args = clusterAnnounceCommand(containers[0].Args) + containers[0].Command, containers[0].Args = clusterAnnounceCommand(containers[0].Args, p.TLSSecretName != "") } podSpec := corev1.PodSpec{ @@ -374,12 +375,20 @@ func buildRestrictedContainerSecurityContext() *corev1.SecurityContext { // CLI flag 가 valkey.conf 보다 우선하므로 ConfigMap 의 cluster-* directive 는 // 유효하게 유지된다. `exec` 로 PID 1 을 valkey-server 로 교체해 signal/graceful // shutdown 의미를 보존한다. -func clusterAnnounceCommand(serverArgs []string) ([]string, []string) { +func clusterAnnounceCommand(serverArgs []string, tlsEnabled bool) ([]string, []string) { + // cluster-bus 는 tls-cluster 시 tls-port(6380)+10000, 아니면 port(6379)+10000 에 + // listen 한다. announce-bus-port 가 실제 listen 포트와 어긋나면 peer 가 닫힌 포트로 + // gossip 을 시도해 모든 노드가 fail?/disconnected → cluster_state:fail, slots_ok:0. + // (TLS 클러스터에서 PortClusterBus(16379) 를 그대로 announce 하던 회귀 사고.) + busPort := PortClient + 10000 + if tlsEnabled { + busPort = PortTLS + 10000 + } parts := append([]string{"exec", "valkey-server"}, serverArgs...) parts = append(parts, "--cluster-announce-ip", `"$POD_IP"`, "--cluster-announce-port", fmt.Sprintf("%d", PortClient), - "--cluster-announce-bus-port", fmt.Sprintf("%d", PortClusterBus), + "--cluster-announce-bus-port", fmt.Sprintf("%d", busPort), ) return []string{"sh", "-c", strings.Join(parts, " ")}, nil }