diff --git a/internal/controller/tls_cert_hash.go b/internal/controller/tls_cert_hash.go new file mode 100644 index 00000000..ddd17677 --- /dev/null +++ b/internal/controller/tls_cert_hash.go @@ -0,0 +1,54 @@ +/* +Copyright 2026 Keiailab. + +Licensed under the MIT License. See the LICENSE file for details. +*/ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// hashTLSSecret — TLS Secret data (tls.crt + tls.key + ca.crt) 의 SHA256 (hex) +// hash. STS PodTemplate annotation `cache.keiailab.io/tls-cert-hash` 으로 주입되어 +// *cert rotation 추적*. cert-manager 가 leaf/CA 를 재발급(예: CA rotation)하면: +// +// - 다음 reconcile 에서 Secret data 가 변경됨 +// - 새 hash 가 STS Template 에 set +// - STS controller 가 PodTemplate 변경 감지 → rolling update 시작 +// - 모든 pod 가 *새 cert/CA* 를 마운트한 채 재시작 (프로세스가 디스크 cert 를 +// 시작 시점에만 read 하는 결함을 우회 — old cert 로 서빙하는 mTLS 단절 방지) +// +// secretName 이 빈 문자열이거나 Secret 미존재 시 빈 문자열 반환 → annotation +// 미설정 (TLS rotation 추적 비활성). Secret 은 곧 cert-manager 가 채우므로 +// 다음 reconcile 에서 hash 가 채워진다 (fail-soft). +// +// 보안: SHA256 의 *hash* 만 노출. 원본 cert/key 는 K8s API(annotation)에 노출 안 됨. +func hashTLSSecret(ctx context.Context, c client.Client, namespace, secretName string) (string, error) { + if secretName == "" { + return "", nil + } + s := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, s); err != nil { + if apierrors.IsNotFound(err) { + // cert-manager 가 아직 Secret 을 만들지 않음 — fail-soft. + return "", nil + } + return "", err + } + h := sha256.New() + // 결정적 순서로 누적 (map 순회 비결정성 회피). + for _, key := range []string{"tls.crt", "tls.key", "ca.crt"} { + // key 자체도 누적해 빈 값/누락을 구분 (예: ca.crt 만 변경되어도 hash 변경). + h.Write([]byte(key)) + h.Write(s.Data[key]) + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/controller/tls_cert_hash_test.go b/internal/controller/tls_cert_hash_test.go new file mode 100644 index 00000000..2b05e038 --- /dev/null +++ b/internal/controller/tls_cert_hash_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2026 Keiailab. + +Licensed under the MIT License. See the LICENSE file for details. +*/ +package controller + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newTLSSecret(name string, crt, key, ca string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns"}, + Data: map[string][]byte{ + "tls.crt": []byte(crt), + "tls.key": []byte(key), + "ca.crt": []byte(ca), + }, + } +} + +// Defect ①: empty secretName → empty hash (annotation 미설정). +func TestHashTLSSecret_emptyName(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + h, err := hashTLSSecret(testCtx(), c, "ns", "") + if err != nil { + t.Fatalf("err: %v", err) + } + if h != "" { + t.Errorf("empty secretName should produce empty hash, got %q", h) + } +} + +// Secret 미존재 → fail-soft (빈 hash, no error). cert-manager 가 아직 미발급한 경우. +func TestHashTLSSecret_notFound(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + h, err := hashTLSSecret(testCtx(), c, "ns", "missing") + if err != nil { + t.Fatalf("missing secret should be fail-soft, got err: %v", err) + } + if h != "" { + t.Errorf("missing secret → empty hash, got %q", h) + } +} + +// 존재하는 Secret → 비어 있지 않은 deterministic hash. +func TestHashTLSSecret_stableAndDeterministic(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + sec := newTLSSecret("vk-tls", "CERT", "KEY", "CA") + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sec).Build() + h1, err := hashTLSSecret(testCtx(), c, "ns", "vk-tls") + if err != nil { + t.Fatalf("err: %v", err) + } + if h1 == "" { + t.Fatal("expected non-empty hash for existing secret") + } + h2, _ := hashTLSSecret(testCtx(), c, "ns", "vk-tls") + if h1 != h2 { + t.Errorf("hash must be deterministic: %q != %q", h1, h2) + } +} + +// Defect ① 핵심: Secret 내용 변경(예: CA rotation) 시 hash 가 변경되어야 함. +func TestHashTLSSecret_changesWhenContentChanges(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + build := func(crt, key, ca string) string { + sec := newTLSSecret("vk-tls", crt, key, ca) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sec).Build() + h, err := hashTLSSecret(testCtx(), c, "ns", "vk-tls") + if err != nil { + t.Fatalf("err: %v", err) + } + return h + } + + base := build("CERT", "KEY", "CA") + cases := map[string]string{ + "cert rotated": build("CERT2", "KEY", "CA"), + "key rotated": build("CERT", "KEY2", "CA"), + "CA rotated": build("CERT", "KEY", "CA2"), + } + for name, h := range cases { + if h == base { + t.Errorf("%s: hash must change when secret content changes (got same as base)", name) + } + } +} diff --git a/internal/controller/valkey_controller.go b/internal/controller/valkey_controller.go index b1a0f777..9b7f28b4 100644 --- a/internal/controller/valkey_controller.go +++ b/internal/controller/valkey_controller.go @@ -228,6 +228,12 @@ func (r *ValkeyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr case v.Spec.TLS.CertManager != nil && v.Spec.TLS.CertManager.IssuerRef.Name != "": stsParams.TLSSecretName = resources.CertificateSecretName(v.Name) } + // TLS cert hash — cert-manager rotation 시 pod rolling restart 트리거. + tlsHash, err := hashTLSSecret(ctx, r.Client, v.Namespace, stsParams.TLSSecretName) + if err != nil { + return applyErrorCondition(ctx, r.Client, v, "TLSCertHash", err, r.Recorder) + } + stsParams.TLSCertHash = tlsHash } if v.Spec.Monitoring != nil && v.Spec.Monitoring.Enabled { stsParams.ExporterImg = exporterImage(v.Spec.Monitoring) diff --git a/internal/controller/valkeycluster_controller.go b/internal/controller/valkeycluster_controller.go index 3b9c7470..1f07df11 100644 --- a/internal/controller/valkeycluster_controller.go +++ b/internal/controller/valkeycluster_controller.go @@ -235,6 +235,12 @@ func (r *ValkeyClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques case vc.Spec.TLS.CertManager != nil && vc.Spec.TLS.CertManager.IssuerRef.Name != "": stsParams.TLSSecretName = resources.CertificateSecretName(vc.Name) } + // TLS cert hash — cert-manager rotation 시 pod rolling restart 트리거. + tlsHash, err := hashTLSSecret(ctx, r.Client, vc.Namespace, stsParams.TLSSecretName) + if err != nil { + return applyErrorCondition(ctx, r.Client, vc, "TLSCertHash", err, r.Recorder) + } + stsParams.TLSCertHash = tlsHash } if vc.Spec.Monitoring != nil && vc.Spec.Monitoring.Enabled { stsParams.ExporterImg = exporterImage(vc.Spec.Monitoring) diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index fe8a3c08..94316855 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -52,6 +52,14 @@ type STSParams struct { // 빈 문자열이면 annotation 미설정 (rotation 추적 비활성). AuthSecretHash string + // TLSCertHash — TLS Secret data (tls.crt + tls.key + ca.crt) 의 SHA256 (hex) + // hash. PodTemplate 의 annotation `cache.keiailab.io/tls-cert-hash` 로 주입되어, + // cert-manager 가 cert/CA 를 재발급(rotation)해 Secret data 가 바뀌면 hash 변경 + // → STS rolling update 가 자동 트리거 → pod 들이 새 cert 를 마운트한 채 재시작. + // valkey-server 가 디스크 cert 를 시작 시점에만 read 하는 결함을 우회한다. + // 빈 문자열이면 annotation 미설정 (TLS 미활성 또는 Secret 미준비). + TLSCertHash string + // Modules — Valkey module 목록. 비어 있지 않으면 BuildModuleInitContainers 가 // init-container(.so 를 공유 emptyDir 로 cp) + volume + --loadmodule args 를 생성한다. Modules []cachev1alpha1.ModuleSpec @@ -94,9 +102,15 @@ func BuildStatefulSet(p STSParams) *appsv1.StatefulSet { // commons storageclass.Normalize — 빈 값 = nil (cluster default StorageClass). dataPVC.Spec.StorageClassName = storageclass.Normalize(storageClass) - envFromPassword := []corev1.EnvVar{} + // POD_IP — downward API 로 pod 의 실제 IP 주입. cluster mode 에서 + // cluster-announce-ip 로 사용 (Defect ②: pod 재시작 후 새 IP 를 gossip 에 + // 광고하지 못해 멤버십이 깨지는 결함 방지). 비-cluster 모드에서도 무해. + podEnv := []corev1.EnvVar{{ + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}}, + }} if p.PasswordRef != nil { - envFromPassword = append(envFromPassword, corev1.EnvVar{ + podEnv = append(podEnv, corev1.EnvVar{ Name: "VALKEY_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: p.PasswordRef}, }) @@ -114,7 +128,7 @@ func BuildStatefulSet(p STSParams) *appsv1.StatefulSet { Ports: []corev1.ContainerPort{ {Name: "client", ContainerPort: PortClient, Protocol: corev1.ProtocolTCP}, }, - Env: envFromPassword, + Env: podEnv, Resources: p.Resources, SecurityContext: buildRestrictedContainerSecurityContext(), VolumeMounts: append([]corev1.VolumeMount{ @@ -203,6 +217,16 @@ func BuildStatefulSet(p STSParams) *appsv1.StatefulSet { volumes = append(volumes, moduleVol) } + // Cluster mode: cluster-announce-ip 를 pod 의 실제 IP 로 광고 (Defect ②). + // cluster-announce-ip 는 startup 시점 literal 이어야 하므로 ConfigMap 이 아니라 + // 컨테이너 command 에서 $POD_IP 를 셸 확장한다. CLI flag 가 valkey.conf 보다 + // 우선하므로 ConfigMap 의 기타 directive 는 그대로 유효. + // announce-port=client(6379/tls 6380 무관 — 6379 는 평문/내부 dial 기준), + // announce-bus-port=cluster-bus(16379). + if p.ClusterMode { + containers[0].Command, containers[0].Args = clusterAnnounceCommand(containers[0].Args) + } + podSpec := corev1.PodSpec{ Containers: containers, InitContainers: moduleInits, @@ -292,18 +316,25 @@ func BuildStatefulSet(p STSParams) *appsv1.StatefulSet { // 로 재시작. const AnnotationAuthSecretHash = "cache.keiailab.io/auth-secret-hash" +// AnnotationTLSCertHash — pod template annotation 키. TLS Secret(cert/CA) +// rotation 추적용. cert-manager 가 cert 를 재발급해 hash 가 바뀌면 STS +// RollingUpdate 가 자동 발동되어 모든 pod 가 새 cert 로 재시작. +const AnnotationTLSCertHash = "cache.keiailab.io/tls-cert-hash" + func podTemplateAnnotations(p STSParams) map[string]string { annotations := map[string]string{} if p.Pod != nil { maps.Copy(annotations, p.Pod.Annotations) } - if p.AuthSecretHash == "" { - if len(annotations) == 0 { - return nil - } - return annotations + if p.AuthSecretHash != "" { + annotations[AnnotationAuthSecretHash] = p.AuthSecretHash + } + if p.TLSCertHash != "" { + annotations[AnnotationTLSCertHash] = p.TLSCertHash + } + if len(annotations) == 0 { + return nil } - annotations[AnnotationAuthSecretHash] = p.AuthSecretHash return annotations } @@ -333,6 +364,26 @@ func buildRestrictedContainerSecurityContext() *corev1.SecurityContext { ) } +// clusterAnnounceCommand — cluster mode 컨테이너 command 를 셸 래핑하여 +// cluster-announce-ip 를 $POD_IP(downward API)로 확장한다. 입력은 기존 +// valkey-server args (config path + --loadmodule 등). 반환은 (command, args) +// 쌍: command 는 `sh -c '...'`, args 는 nil (셸 명령에 모두 인라인). +// +// cluster-announce-ip 는 valkey-server 시작 시점에 literal 이어야 하므로 +// ConfigMap directive 로 표현할 수 없다 (pod IP 는 ConfigMap 렌더 시점 미확정). +// CLI flag 가 valkey.conf 보다 우선하므로 ConfigMap 의 cluster-* directive 는 +// 유효하게 유지된다. `exec` 로 PID 1 을 valkey-server 로 교체해 signal/graceful +// shutdown 의미를 보존한다. +func clusterAnnounceCommand(serverArgs []string) ([]string, []string) { + parts := append([]string{"exec", "valkey-server"}, serverArgs...) + parts = append(parts, + "--cluster-announce-ip", `"$POD_IP"`, + "--cluster-announce-port", fmt.Sprintf("%d", PortClient), + "--cluster-announce-bus-port", fmt.Sprintf("%d", PortClusterBus), + ) + return []string{"sh", "-c", strings.Join(parts, " ")}, nil +} + // PortIntOrString — helper for Probe/Service ports. func PortIntOrString(p int32) intstr.IntOrString { return intstr.FromInt(int(p)) } diff --git a/internal/resources/sts_annotation_test.go b/internal/resources/sts_annotation_test.go index 4c421b71..725ffa97 100644 --- a/internal/resources/sts_annotation_test.go +++ b/internal/resources/sts_annotation_test.go @@ -6,8 +6,10 @@ Licensed under the MIT License. See the LICENSE file for details. package resources import ( + "strings" "testing" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -56,3 +58,108 @@ func TestBuildStatefulSet_no_annotation_when_hash_empty(t *testing.T) { sts.Spec.Template.Annotations) } } + +// Defect ①: TLS cert hash annotation. cert-manager rotation 시 hash 변경 → +// PodTemplate 변경 → STS rolling update. +func TestPodTemplateAnnotations_tls_hash_set(t *testing.T) { + p := minimalSTSParams() + p.TLSCertHash = "tlshash123" + a := podTemplateAnnotations(p) + if a[AnnotationTLSCertHash] != "tlshash123" { + t.Errorf("tls annotation: %v", a) + } +} + +func TestPodTemplateAnnotations_both_hashes_independent(t *testing.T) { + p := minimalSTSParams() + p.AuthSecretHash = "auth1" + p.TLSCertHash = "tls1" + a := podTemplateAnnotations(p) + if a[AnnotationAuthSecretHash] != "auth1" || a[AnnotationTLSCertHash] != "tls1" { + t.Errorf("both hashes should be set independently: %v", a) + } +} + +func TestBuildStatefulSet_tls_hash_propagates_and_changes(t *testing.T) { + p := minimalSTSParams() + p.TLSCertHash = "hashA" + sts1 := BuildStatefulSet(p) + if got := sts1.Spec.Template.Annotations[AnnotationTLSCertHash]; got != "hashA" { + t.Fatalf("tls hash annotation = %q, want hashA", got) + } + // Secret content 변경 → hash 변경 → PodTemplate annotation 변경 (rolling update 트리거). + p.TLSCertHash = "hashB" + sts2 := BuildStatefulSet(p) + if got := sts2.Spec.Template.Annotations[AnnotationTLSCertHash]; got != "hashB" { + t.Fatalf("tls hash annotation after change = %q, want hashB", got) + } + if sts1.Spec.Template.Annotations[AnnotationTLSCertHash] == sts2.Spec.Template.Annotations[AnnotationTLSCertHash] { + t.Error("PodTemplate annotation must differ when TLS secret content changes") + } +} + +func TestBuildStatefulSet_no_tls_annotation_when_hash_empty(t *testing.T) { + p := minimalSTSParams() + sts := BuildStatefulSet(p) + if _, ok := sts.Spec.Template.Annotations[AnnotationTLSCertHash]; ok { + t.Errorf("tls annotation should not be set when hash empty: %v", + sts.Spec.Template.Annotations) + } +} + +// Defect ②: cluster-announce-ip. POD_IP downward API env + cluster mode command +// 에서 $POD_IP 셸 확장. +func TestBuildStatefulSet_pod_ip_env_via_downward_api(t *testing.T) { + p := minimalSTSParams() + sts := BuildStatefulSet(p) + c := sts.Spec.Template.Spec.Containers[0] + var podIPEnv *corev1.EnvVar + for i := range c.Env { + if c.Env[i].Name == "POD_IP" { + podIPEnv = &c.Env[i] + } + } + if podIPEnv == nil { + t.Fatalf("POD_IP env missing: %v", c.Env) + } + if podIPEnv.ValueFrom == nil || podIPEnv.ValueFrom.FieldRef == nil || + podIPEnv.ValueFrom.FieldRef.FieldPath != "status.podIP" { + t.Errorf("POD_IP must come from downward API status.podIP, got %+v", podIPEnv.ValueFrom) + } +} + +func TestBuildStatefulSet_cluster_announce_command(t *testing.T) { + p := minimalSTSParams() + p.ClusterMode = true + sts := BuildStatefulSet(p) + c := sts.Spec.Template.Spec.Containers[0] + if len(c.Command) != 3 || c.Command[0] != "sh" || c.Command[1] != "-c" { + t.Fatalf("cluster mode command should be sh -c ..., got %v", c.Command) + } + shCmd := c.Command[2] + if !strings.Contains(shCmd, "valkey-server") { + t.Errorf("command must exec valkey-server: %s", shCmd) + } + if !strings.Contains(shCmd, `--cluster-announce-ip "$POD_IP"`) { + t.Errorf("cluster-announce-ip $POD_IP missing: %s", shCmd) + } + if !strings.Contains(shCmd, "--cluster-announce-port 6379") { + t.Errorf("cluster-announce-port missing: %s", shCmd) + } + if !strings.Contains(shCmd, "--cluster-announce-bus-port 16379") { + t.Errorf("cluster-announce-bus-port missing: %s", shCmd) + } + // config path 가 보존되어야 함. + if !strings.Contains(shCmd, ConfigMapMountPath+"/"+ConfigFileName) { + t.Errorf("config path lost in cluster command: %s", shCmd) + } +} + +func TestBuildStatefulSet_standalone_keeps_plain_command(t *testing.T) { + p := minimalSTSParams() + sts := BuildStatefulSet(p) + c := sts.Spec.Template.Spec.Containers[0] + if len(c.Command) != 1 || c.Command[0] != "valkey-server" { + t.Errorf("standalone command should stay valkey-server, got %v", c.Command) + } +}