diff --git a/api/v1alpha1/conditions.go b/api/v1alpha1/conditions.go index 7dfc5a8..08c0ccb 100644 --- a/api/v1alpha1/conditions.go +++ b/api/v1alpha1/conditions.go @@ -36,6 +36,11 @@ const ( ConditionUpgradeBlocked = "UpgradeBlocked" // ConditionShardCountLocked indicates the immutable shard count is locked in. ConditionShardCountLocked = "ShardCountLocked" + // ConditionProxyDeployed indicates the s2s-proxy Deployment is available. + ConditionProxyDeployed = "ProxyDeployed" + // ConditionRemoteClusterRegistered indicates the local Temporal registered the + // peer as a remote cluster via the local proxy. + ConditionRemoteClusterRegistered = "RemoteClusterRegistered" ) // Condition reasons reported on Temporal resource status. @@ -74,4 +79,14 @@ const ( ReasonReplicationDrift = "ReplicationConfigDrift" // ReasonActiveClusterInvalid indicates the active cluster selection is invalid. ReasonActiveClusterInvalid = "ActiveClusterInvalid" + // ReasonProxyNotReady indicates the s2s-proxy Deployment is not yet available. + ReasonProxyNotReady = "ProxyNotReady" + // ReasonProxyReady indicates the s2s-proxy Deployment is available. + ReasonProxyReady = "ProxyReady" + // ReasonClusterNotReady indicates the referenced cluster is not yet ready. + ReasonClusterNotReady = "ClusterNotReady" + // ReasonMTLSNotReady indicates mTLS material is not yet provisioned. + ReasonMTLSNotReady = "MTLSNotReady" + // ReasonRegistrationFailed indicates remote cluster registration failed. + ReasonRegistrationFailed = "RegistrationFailed" ) diff --git a/api/v1alpha1/temporalclusterproxy_types.go b/api/v1alpha1/temporalclusterproxy_types.go new file mode 100644 index 0000000..5fe93e9 --- /dev/null +++ b/api/v1alpha1/temporalclusterproxy_types.go @@ -0,0 +1,246 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Mux roles for ProxyMux.Role. +const ( + // ProxyRoleServer opens a listening mux port (the exposed side). + ProxyRoleServer = "server" + // ProxyRoleClient dials out to a remote mux-server (never opens a port). + ProxyRoleClient = "client" +) + +// TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link +// to one replication peer over an s2s-proxy mux connection. +type TemporalClusterProxySpec struct { + // LocalClusterRef references the local operator-managed TemporalCluster this + // proxy fronts. Its frontend address and issuer CA are resolved automatically. + LocalClusterRef ClusterReference `json:"localClusterRef"` + + // LocalClusterName overrides the replication-group name of the local cluster. + // Defaults to the referenced cluster's clusterMetadata.currentClusterName. + // +optional + LocalClusterName string `json:"localClusterName,omitempty"` + + // Peer is the remote replication cluster on the far side of the mux. + Peer ProxyPeer `json:"peer"` + + // Mux configures the s2s-proxy multiplexed transport. + Mux ProxyMux `json:"mux"` + + // Translation optionally renames namespaces and search attributes in-flight. + // +optional + Translation *ProxyTranslation `json:"translation,omitempty"` + + // FailoverVersionIncrement optionally translates failover-version increments. + // +optional + FailoverVersionIncrement *ProxyFailoverVersionIncrement `json:"failoverVersionIncrement,omitempty"` + + // ACL optionally restricts the admin methods and namespaces the proxy relays. + // +optional + ACL *ProxyACL `json:"acl,omitempty"` + + // Image overrides the pinned s2s-proxy image. + // +optional + Image string `json:"image,omitempty"` +} + +// ProxyPeer identifies the remote replication cluster reached over the mux. +type ProxyPeer struct { + // Name is the remote replication cluster name (== its currentClusterName). + Name string `json:"name"` + + // ClusterRef optionally references an operator-managed remote TemporalCluster. + // It is used only to reuse the peer's issuer CA when available. + // +optional + ClusterRef *ClusterReference `json:"clusterRef,omitempty"` + + // EnableConnection toggles replication without deleting the CR. + // +kubebuilder:default=true + // +optional + EnableConnection *bool `json:"enableConnection,omitempty"` +} + +// ProxyMux configures the s2s-proxy mux transport for one link. +type ProxyMux struct { + // Role selects whether this proxy opens a port (server) or dials out (client). + // +kubebuilder:validation:Enum=server;client + Role string `json:"role"` + + // Server configures the listening side. Required when role=server. + // +optional + Server *ProxyMuxServer `json:"server,omitempty"` + + // Client configures the dialing side. Required when role=client. + // +optional + Client *ProxyMuxClient `json:"client,omitempty"` + + // MuxCount is the number of multiplexed sessions. Defaults to the upstream default. + // +kubebuilder:validation:Minimum=1 + // +optional + MuxCount *int32 `json:"muxCount,omitempty"` + + // TLS configures the mux mTLS material. + TLS ProxyMuxTLS `json:"tls"` +} + +// ProxyMuxServer configures a mux-server (listening) proxy. +type ProxyMuxServer struct { + // ListenPort is the port the mux listens on. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + ListenPort int32 `json:"listenPort"` + + // Exposure controls how the mux port is exposed (ClusterIP/NodePort/LoadBalancer). + // +optional + Exposure *ServiceExposureSpec `json:"exposure,omitempty"` +} + +// ProxyMuxClient configures a mux-client (dialing) proxy. +type ProxyMuxClient struct { + // ServerAddress is the reachable host:port of the remote mux-server. + ServerAddress string `json:"serverAddress"` +} + +// ProxyMuxTLS configures the mux mTLS material for one side. +type ProxyMuxTLS struct { + // Provider selects how this side's mux certificate is sourced. + // +kubebuilder:validation:Enum=cert-manager;secret + // +kubebuilder:default=cert-manager + // +optional + Provider string `json:"provider,omitempty"` + + // IssuerRef mints this side's mux certificate. Required when provider=cert-manager. + // +optional + IssuerRef *IssuerReference `json:"issuerRef,omitempty"` + + // SecretRef supplies BYO cert/key/CA. Required when provider=secret. + // +optional + SecretRef *SecretReference `json:"secretRef,omitempty"` + + // PeerCARef supplies the remote side's CA to trust. When unset the CA bundle + // from this side's own material is used (shared-issuer case). + // +optional + PeerCARef *SecretReference `json:"peerCARef,omitempty"` + + // CAServerName is the TLS server name a mux-client verifies against the + // remote mux-server's certificate. s2s-proxy requires it on the client side + // unless SkipCAVerification is true. When empty for a client, the operator + // derives it from the host portion of mux.client.serverAddress (which matches + // the server proxy's certificate SANs for operator-managed peers). Ignored + // for the server role. + // +optional + CAServerName string `json:"caServerName,omitempty"` + + // SkipCAVerification disables CA verification of the mux peer's certificate. + // Leave unset (false) for real mTLS; set true only for testing or when a + // verifiable server name is unavailable. + // +optional + SkipCAVerification *bool `json:"skipCAVerification,omitempty"` +} + +// ProxyTranslation renames namespaces and search attributes in-flight. +type ProxyTranslation struct { + // +optional + Namespaces []ProxyNamespaceMapping `json:"namespaces,omitempty"` + // +optional + SearchAttributes []ProxySearchAttributeMapping `json:"searchAttributes,omitempty"` +} + +// ProxyNamespaceMapping maps a local namespace name to a remote one. +type ProxyNamespaceMapping struct { + Local string `json:"local"` + Remote string `json:"remote"` +} + +// ProxySearchAttributeMapping maps search-attribute field names for a namespace. +type ProxySearchAttributeMapping struct { + Namespace string `json:"namespace"` + Mappings []ProxyFieldMapping `json:"mappings"` +} + +// ProxyFieldMapping maps a local search-attribute field name to a remote one. +type ProxyFieldMapping struct { + LocalFieldName string `json:"localFieldName"` + RemoteFieldName string `json:"remoteFieldName"` +} + +// ProxyFailoverVersionIncrement translates failover-version increments across the link. +type ProxyFailoverVersionIncrement struct { + Local int64 `json:"local"` + Remote int64 `json:"remote"` +} + +// ProxyACL restricts what the proxy relays. +type ProxyACL struct { + // +optional + AllowedNamespaces []string `json:"allowedNamespaces,omitempty"` + // AllowedAdminMethods defaults to the standard replication allowlist when empty. + // +optional + AllowedAdminMethods []string `json:"allowedAdminMethods,omitempty"` +} + +// TemporalClusterProxyStatus is the observed state. +type TemporalClusterProxyStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // ProxyEndpoint reports the exposed mux address (server role) to hand to the peer. + // +optional + ProxyEndpoint string `json:"proxyEndpoint,omitempty"` + + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=tcproxy +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Role",type=string,JSONPath=`.spec.mux.role` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// TemporalClusterProxy is the Schema for the temporalclusterproxies API. +type TemporalClusterProxy struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + // +required + Spec TemporalClusterProxySpec `json:"spec"` + // +optional + Status TemporalClusterProxyStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// TemporalClusterProxyList contains a list of TemporalClusterProxy. +type TemporalClusterProxyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []TemporalClusterProxy `json:"items"` +} + +func init() { + registerType(&TemporalClusterProxy{}, &TemporalClusterProxyList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9688934..0fb53c7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -793,6 +793,249 @@ func (in *PodTemplateOverride) DeepCopy() *PodTemplateOverride { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyACL) DeepCopyInto(out *ProxyACL) { + *out = *in + if in.AllowedNamespaces != nil { + in, out := &in.AllowedNamespaces, &out.AllowedNamespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedAdminMethods != nil { + in, out := &in.AllowedAdminMethods, &out.AllowedAdminMethods + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyACL. +func (in *ProxyACL) DeepCopy() *ProxyACL { + if in == nil { + return nil + } + out := new(ProxyACL) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyFailoverVersionIncrement) DeepCopyInto(out *ProxyFailoverVersionIncrement) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyFailoverVersionIncrement. +func (in *ProxyFailoverVersionIncrement) DeepCopy() *ProxyFailoverVersionIncrement { + if in == nil { + return nil + } + out := new(ProxyFailoverVersionIncrement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyFieldMapping) DeepCopyInto(out *ProxyFieldMapping) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyFieldMapping. +func (in *ProxyFieldMapping) DeepCopy() *ProxyFieldMapping { + if in == nil { + return nil + } + out := new(ProxyFieldMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyMux) DeepCopyInto(out *ProxyMux) { + *out = *in + if in.Server != nil { + in, out := &in.Server, &out.Server + *out = new(ProxyMuxServer) + (*in).DeepCopyInto(*out) + } + if in.Client != nil { + in, out := &in.Client, &out.Client + *out = new(ProxyMuxClient) + **out = **in + } + if in.MuxCount != nil { + in, out := &in.MuxCount, &out.MuxCount + *out = new(int32) + **out = **in + } + in.TLS.DeepCopyInto(&out.TLS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMux. +func (in *ProxyMux) DeepCopy() *ProxyMux { + if in == nil { + return nil + } + out := new(ProxyMux) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyMuxClient) DeepCopyInto(out *ProxyMuxClient) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMuxClient. +func (in *ProxyMuxClient) DeepCopy() *ProxyMuxClient { + if in == nil { + return nil + } + out := new(ProxyMuxClient) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyMuxServer) DeepCopyInto(out *ProxyMuxServer) { + *out = *in + if in.Exposure != nil { + in, out := &in.Exposure, &out.Exposure + *out = new(ServiceExposureSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMuxServer. +func (in *ProxyMuxServer) DeepCopy() *ProxyMuxServer { + if in == nil { + return nil + } + out := new(ProxyMuxServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyMuxTLS) DeepCopyInto(out *ProxyMuxTLS) { + *out = *in + if in.IssuerRef != nil { + in, out := &in.IssuerRef, &out.IssuerRef + *out = new(IssuerReference) + **out = **in + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(SecretReference) + **out = **in + } + if in.PeerCARef != nil { + in, out := &in.PeerCARef, &out.PeerCARef + *out = new(SecretReference) + **out = **in + } + if in.SkipCAVerification != nil { + in, out := &in.SkipCAVerification, &out.SkipCAVerification + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMuxTLS. +func (in *ProxyMuxTLS) DeepCopy() *ProxyMuxTLS { + if in == nil { + return nil + } + out := new(ProxyMuxTLS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyNamespaceMapping) DeepCopyInto(out *ProxyNamespaceMapping) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyNamespaceMapping. +func (in *ProxyNamespaceMapping) DeepCopy() *ProxyNamespaceMapping { + if in == nil { + return nil + } + out := new(ProxyNamespaceMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyPeer) DeepCopyInto(out *ProxyPeer) { + *out = *in + if in.ClusterRef != nil { + in, out := &in.ClusterRef, &out.ClusterRef + *out = new(ClusterReference) + **out = **in + } + if in.EnableConnection != nil { + in, out := &in.EnableConnection, &out.EnableConnection + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyPeer. +func (in *ProxyPeer) DeepCopy() *ProxyPeer { + if in == nil { + return nil + } + out := new(ProxyPeer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxySearchAttributeMapping) DeepCopyInto(out *ProxySearchAttributeMapping) { + *out = *in + if in.Mappings != nil { + in, out := &in.Mappings, &out.Mappings + *out = make([]ProxyFieldMapping, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxySearchAttributeMapping. +func (in *ProxySearchAttributeMapping) DeepCopy() *ProxySearchAttributeMapping { + if in == nil { + return nil + } + out := new(ProxySearchAttributeMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyTranslation) DeepCopyInto(out *ProxyTranslation) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]ProxyNamespaceMapping, len(*in)) + copy(*out, *in) + } + if in.SearchAttributes != nil { + in, out := &in.SearchAttributes, &out.SearchAttributes + *out = make([]ProxySearchAttributeMapping, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyTranslation. +func (in *ProxyTranslation) DeepCopy() *ProxyTranslation { + if in == nil { + return nil + } + out := new(ProxyTranslation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RetryPolicySpec) DeepCopyInto(out *RetryPolicySpec) { *out = *in @@ -1597,6 +1840,120 @@ func (in *TemporalClusterList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalClusterProxy) DeepCopyInto(out *TemporalClusterProxy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalClusterProxy. +func (in *TemporalClusterProxy) DeepCopy() *TemporalClusterProxy { + if in == nil { + return nil + } + out := new(TemporalClusterProxy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TemporalClusterProxy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalClusterProxyList) DeepCopyInto(out *TemporalClusterProxyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TemporalClusterProxy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalClusterProxyList. +func (in *TemporalClusterProxyList) DeepCopy() *TemporalClusterProxyList { + if in == nil { + return nil + } + out := new(TemporalClusterProxyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TemporalClusterProxyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalClusterProxySpec) DeepCopyInto(out *TemporalClusterProxySpec) { + *out = *in + out.LocalClusterRef = in.LocalClusterRef + in.Peer.DeepCopyInto(&out.Peer) + in.Mux.DeepCopyInto(&out.Mux) + if in.Translation != nil { + in, out := &in.Translation, &out.Translation + *out = new(ProxyTranslation) + (*in).DeepCopyInto(*out) + } + if in.FailoverVersionIncrement != nil { + in, out := &in.FailoverVersionIncrement, &out.FailoverVersionIncrement + *out = new(ProxyFailoverVersionIncrement) + **out = **in + } + if in.ACL != nil { + in, out := &in.ACL, &out.ACL + *out = new(ProxyACL) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalClusterProxySpec. +func (in *TemporalClusterProxySpec) DeepCopy() *TemporalClusterProxySpec { + if in == nil { + return nil + } + out := new(TemporalClusterProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemporalClusterProxyStatus) DeepCopyInto(out *TemporalClusterProxyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalClusterProxyStatus. +func (in *TemporalClusterProxyStatus) DeepCopy() *TemporalClusterProxyStatus { + if in == nil { + return nil + } + out := new(TemporalClusterProxyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TemporalClusterSpec) DeepCopyInto(out *TemporalClusterSpec) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index fdd3975..983be26 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -249,6 +249,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "TemporalClusterConnection") os.Exit(1) } + if err := (&controller.TemporalClusterProxyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TemporalClusterProxy") + os.Exit(1) + } webhooksEnabled := os.Getenv("ENABLE_WEBHOOKS") != "false" if webhooksEnabled { if err := webhookv1alpha1.SetupTemporalClusterWebhookWithManager(mgr); err != nil { @@ -280,6 +287,12 @@ func main() { os.Exit(1) } } + if webhooksEnabled { + if err := webhookv1alpha1.SetupTemporalClusterProxyWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "TemporalClusterProxy") + os.Exit(1) + } + } // +kubebuilder:scaffold:builder uiOpts := ui.Options{ diff --git a/config/crd/bases/temporal.bmor10.com_temporalclusterproxies.yaml b/config/crd/bases/temporal.bmor10.com_temporalclusterproxies.yaml new file mode 100644 index 0000000..078cb1d --- /dev/null +++ b/config/crd/bases/temporal.bmor10.com_temporalclusterproxies.yaml @@ -0,0 +1,424 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: temporalclusterproxies.temporal.bmor10.com +spec: + group: temporal.bmor10.com + names: + kind: TemporalClusterProxy + listKind: TemporalClusterProxyList + plural: temporalclusterproxies + shortNames: + - tcproxy + singular: temporalclusterproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.mux.role + name: Role + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TemporalClusterProxy is the Schema for the temporalclusterproxies + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link + to one replication peer over an s2s-proxy mux connection. + properties: + acl: + description: ACL optionally restricts the admin methods and namespaces + the proxy relays. + properties: + allowedAdminMethods: + description: AllowedAdminMethods defaults to the standard replication + allowlist when empty. + items: + type: string + type: array + allowedNamespaces: + items: + type: string + type: array + type: object + failoverVersionIncrement: + description: FailoverVersionIncrement optionally translates failover-version + increments. + properties: + local: + format: int64 + type: integer + remote: + format: int64 + type: integer + required: + - local + - remote + type: object + image: + description: Image overrides the pinned s2s-proxy image. + type: string + localClusterName: + description: |- + LocalClusterName overrides the replication-group name of the local cluster. + Defaults to the referenced cluster's clusterMetadata.currentClusterName. + type: string + localClusterRef: + description: |- + LocalClusterRef references the local operator-managed TemporalCluster this + proxy fronts. Its frontend address and issuer CA are resolved automatically. + properties: + kind: + default: TemporalCluster + description: Kind selects the referenced object type. + enum: + - TemporalCluster + - TemporalDevServer + type: string + name: + description: Name is the name of the referenced object. + type: string + required: + - name + type: object + mux: + description: Mux configures the s2s-proxy multiplexed transport. + properties: + client: + description: Client configures the dialing side. Required when + role=client. + properties: + serverAddress: + description: ServerAddress is the reachable host:port of the + remote mux-server. + type: string + required: + - serverAddress + type: object + muxCount: + description: MuxCount is the number of multiplexed sessions. Defaults + to the upstream default. + format: int32 + minimum: 1 + type: integer + role: + description: Role selects whether this proxy opens a port (server) + or dials out (client). + enum: + - server + - client + type: string + server: + description: Server configures the listening side. Required when + role=server. + properties: + exposure: + description: Exposure controls how the mux port is exposed + (ClusterIP/NodePort/LoadBalancer). + properties: + annotations: + additionalProperties: + type: string + type: object + type: + default: ClusterIP + description: Service Type string describes ingress methods + for a service + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + listenPort: + description: ListenPort is the port the mux listens on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - listenPort + type: object + tls: + description: TLS configures the mux mTLS material. + properties: + caServerName: + description: |- + CAServerName is the TLS server name a mux-client verifies against the + remote mux-server's certificate. s2s-proxy requires it on the client side + unless SkipCAVerification is true. When empty for a client, the operator + derives it from the host portion of mux.client.serverAddress (which matches + the server proxy's certificate SANs for operator-managed peers). Ignored + for the server role. + type: string + issuerRef: + description: IssuerRef mints this side's mux certificate. + Required when provider=cert-manager. + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + enum: + - Issuer + - ClusterIssuer + type: string + name: + type: string + required: + - name + type: object + peerCARef: + description: |- + PeerCARef supplies the remote side's CA to trust. When unset the CA bundle + from this side's own material is used (shared-issuer case). + properties: + caKey: + description: CAKey is the Secret key holding the CA bundle. + Defaults to "ca.crt". + type: string + certKey: + description: CertKey is the Secret key holding the client + certificate. Defaults to "tls.crt". + type: string + keyKey: + description: KeyKey is the Secret key holding the client + private key. Defaults to "tls.key". + type: string + name: + description: Name is the Secret name. + type: string + required: + - name + type: object + provider: + default: cert-manager + description: Provider selects how this side's mux certificate + is sourced. + enum: + - cert-manager + - secret + type: string + secretRef: + description: SecretRef supplies BYO cert/key/CA. Required + when provider=secret. + properties: + caKey: + description: CAKey is the Secret key holding the CA bundle. + Defaults to "ca.crt". + type: string + certKey: + description: CertKey is the Secret key holding the client + certificate. Defaults to "tls.crt". + type: string + keyKey: + description: KeyKey is the Secret key holding the client + private key. Defaults to "tls.key". + type: string + name: + description: Name is the Secret name. + type: string + required: + - name + type: object + skipCAVerification: + description: |- + SkipCAVerification disables CA verification of the mux peer's certificate. + Leave unset (false) for real mTLS; set true only for testing or when a + verifiable server name is unavailable. + type: boolean + type: object + required: + - role + - tls + type: object + peer: + description: Peer is the remote replication cluster on the far side + of the mux. + properties: + clusterRef: + description: |- + ClusterRef optionally references an operator-managed remote TemporalCluster. + It is used only to reuse the peer's issuer CA when available. + properties: + kind: + default: TemporalCluster + description: Kind selects the referenced object type. + enum: + - TemporalCluster + - TemporalDevServer + type: string + name: + description: Name is the name of the referenced object. + type: string + required: + - name + type: object + enableConnection: + default: true + description: EnableConnection toggles replication without deleting + the CR. + type: boolean + name: + description: Name is the remote replication cluster name (== its + currentClusterName). + type: string + required: + - name + type: object + translation: + description: Translation optionally renames namespaces and search + attributes in-flight. + properties: + namespaces: + items: + description: ProxyNamespaceMapping maps a local namespace name + to a remote one. + properties: + local: + type: string + remote: + type: string + required: + - local + - remote + type: object + type: array + searchAttributes: + items: + description: ProxySearchAttributeMapping maps search-attribute + field names for a namespace. + properties: + mappings: + items: + description: ProxyFieldMapping maps a local search-attribute + field name to a remote one. + properties: + localFieldName: + type: string + remoteFieldName: + type: string + required: + - localFieldName + - remoteFieldName + type: object + type: array + namespace: + type: string + required: + - mappings + - namespace + type: object + type: array + type: object + required: + - localClusterRef + - mux + - peer + type: object + status: + description: TemporalClusterProxyStatus is the observed state. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + proxyEndpoint: + description: ProxyEndpoint reports the exposed mux address (server + role) to hand to the peer. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c72889f..58e1956 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/temporal.bmor10.com_temporalschedules.yaml - bases/temporal.bmor10.com_temporaldevservers.yaml - bases/temporal.bmor10.com_temporalclusterconnections.yaml +- bases/temporal.bmor10.com_temporalclusterproxies.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 065895a..0af5d64 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -121,6 +121,7 @@ rules: resources: - temporalclusterclients - temporalclusterconnections + - temporalclusterproxies - temporalclusters - temporaldevservers - temporalnamespaces @@ -139,6 +140,7 @@ rules: resources: - temporalclusterclients/finalizers - temporalclusterconnections/finalizers + - temporalclusterproxies/finalizers - temporalclusters/finalizers - temporaldevservers/finalizers - temporalnamespaces/finalizers @@ -151,6 +153,7 @@ rules: resources: - temporalclusterclients/status - temporalclusterconnections/status + - temporalclusterproxies/status - temporalclusters/status - temporaldevservers/status - temporalnamespaces/status diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 9b1703c..dc1e58b 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -24,6 +24,26 @@ webhooks: resources: - temporalclusters sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-temporal-bmor10-com-v1alpha1-temporalclusterproxy + failurePolicy: Fail + name: mtemporalclusterproxy-v1alpha1.kb.io + rules: + - apiGroups: + - temporal.bmor10.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - temporalclusterproxies + sideEffects: None --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration @@ -71,6 +91,26 @@ webhooks: resources: - temporalclusterconnections sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-temporal-bmor10-com-v1alpha1-temporalclusterproxy + failurePolicy: Fail + name: vtemporalclusterproxy-v1alpha1.kb.io + rules: + - apiGroups: + - temporal.bmor10.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - temporalclusterproxies + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/dist/chart/templates/crd/temporalclusterproxies.temporal.bmor10.com.yaml b/dist/chart/templates/crd/temporalclusterproxies.temporal.bmor10.com.yaml new file mode 100644 index 0000000..095cc3a --- /dev/null +++ b/dist/chart/templates/crd/temporalclusterproxies.temporal.bmor10.com.yaml @@ -0,0 +1,396 @@ +{{- if .Values.crd.enable }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: temporalclusterproxies.temporal.bmor10.com +spec: + group: temporal.bmor10.com + names: + kind: TemporalClusterProxy + listKind: TemporalClusterProxyList + plural: temporalclusterproxies + shortNames: + - tcproxy + singular: temporalclusterproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.mux.role + name: Role + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TemporalClusterProxy is the Schema for the temporalclusterproxies API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link + to one replication peer over an s2s-proxy mux connection. + properties: + acl: + description: ACL optionally restricts the admin methods and namespaces the proxy relays. + properties: + allowedAdminMethods: + description: AllowedAdminMethods defaults to the standard replication allowlist when empty. + items: + type: string + type: array + allowedNamespaces: + items: + type: string + type: array + type: object + failoverVersionIncrement: + description: FailoverVersionIncrement optionally translates failover-version increments. + properties: + local: + format: int64 + type: integer + remote: + format: int64 + type: integer + required: + - local + - remote + type: object + image: + description: Image overrides the pinned s2s-proxy image. + type: string + localClusterName: + description: |- + LocalClusterName overrides the replication-group name of the local cluster. + Defaults to the referenced cluster's clusterMetadata.currentClusterName. + type: string + localClusterRef: + description: |- + LocalClusterRef references the local operator-managed TemporalCluster this + proxy fronts. Its frontend address and issuer CA are resolved automatically. + properties: + kind: + default: TemporalCluster + description: Kind selects the referenced object type. + enum: + - TemporalCluster + - TemporalDevServer + type: string + name: + description: Name is the name of the referenced object. + type: string + required: + - name + type: object + mux: + description: Mux configures the s2s-proxy multiplexed transport. + properties: + client: + description: Client configures the dialing side. Required when role=client. + properties: + serverAddress: + description: ServerAddress is the reachable host:port of the remote mux-server. + type: string + required: + - serverAddress + type: object + muxCount: + description: MuxCount is the number of multiplexed sessions. Defaults to the upstream default. + format: int32 + minimum: 1 + type: integer + role: + description: Role selects whether this proxy opens a port (server) or dials out (client). + enum: + - server + - client + type: string + server: + description: Server configures the listening side. Required when role=server. + properties: + exposure: + description: Exposure controls how the mux port is exposed (ClusterIP/NodePort/LoadBalancer). + properties: + annotations: + additionalProperties: + type: string + type: object + type: + default: ClusterIP + description: Service Type string describes ingress methods for a service + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + listenPort: + description: ListenPort is the port the mux listens on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - listenPort + type: object + tls: + description: TLS configures the mux mTLS material. + properties: + caServerName: + description: |- + CAServerName is the TLS server name a mux-client verifies against the + remote mux-server's certificate. s2s-proxy requires it on the client side + unless SkipCAVerification is true. When empty for a client, the operator + derives it from the host portion of mux.client.serverAddress (which matches + the server proxy's certificate SANs for operator-managed peers). Ignored + for the server role. + type: string + issuerRef: + description: IssuerRef mints this side's mux certificate. Required when provider=cert-manager. + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + enum: + - Issuer + - ClusterIssuer + type: string + name: + type: string + required: + - name + type: object + peerCARef: + description: |- + PeerCARef supplies the remote side's CA to trust. When unset the CA bundle + from this side's own material is used (shared-issuer case). + properties: + caKey: + description: CAKey is the Secret key holding the CA bundle. Defaults to "ca.crt". + type: string + certKey: + description: CertKey is the Secret key holding the client certificate. Defaults to "tls.crt". + type: string + keyKey: + description: KeyKey is the Secret key holding the client private key. Defaults to "tls.key". + type: string + name: + description: Name is the Secret name. + type: string + required: + - name + type: object + provider: + default: cert-manager + description: Provider selects how this side's mux certificate is sourced. + enum: + - cert-manager + - secret + type: string + secretRef: + description: SecretRef supplies BYO cert/key/CA. Required when provider=secret. + properties: + caKey: + description: CAKey is the Secret key holding the CA bundle. Defaults to "ca.crt". + type: string + certKey: + description: CertKey is the Secret key holding the client certificate. Defaults to "tls.crt". + type: string + keyKey: + description: KeyKey is the Secret key holding the client private key. Defaults to "tls.key". + type: string + name: + description: Name is the Secret name. + type: string + required: + - name + type: object + skipCAVerification: + description: |- + SkipCAVerification disables CA verification of the mux peer's certificate. + Leave unset (false) for real mTLS; set true only for testing or when a + verifiable server name is unavailable. + type: boolean + type: object + required: + - role + - tls + type: object + peer: + description: Peer is the remote replication cluster on the far side of the mux. + properties: + clusterRef: + description: |- + ClusterRef optionally references an operator-managed remote TemporalCluster. + It is used only to reuse the peer's issuer CA when available. + properties: + kind: + default: TemporalCluster + description: Kind selects the referenced object type. + enum: + - TemporalCluster + - TemporalDevServer + type: string + name: + description: Name is the name of the referenced object. + type: string + required: + - name + type: object + enableConnection: + default: true + description: EnableConnection toggles replication without deleting the CR. + type: boolean + name: + description: Name is the remote replication cluster name (== its currentClusterName). + type: string + required: + - name + type: object + translation: + description: Translation optionally renames namespaces and search attributes in-flight. + properties: + namespaces: + items: + description: ProxyNamespaceMapping maps a local namespace name to a remote one. + properties: + local: + type: string + remote: + type: string + required: + - local + - remote + type: object + type: array + searchAttributes: + items: + description: ProxySearchAttributeMapping maps search-attribute field names for a namespace. + properties: + mappings: + items: + description: ProxyFieldMapping maps a local search-attribute field name to a remote one. + properties: + localFieldName: + type: string + remoteFieldName: + type: string + required: + - localFieldName + - remoteFieldName + type: object + type: array + namespace: + type: string + required: + - mappings + - namespace + type: object + type: array + type: object + required: + - localClusterRef + - mux + - peer + type: object + status: + description: TemporalClusterProxyStatus is the observed state. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + proxyEndpoint: + description: ProxyEndpoint reports the exposed mux address (server role) to hand to the peer. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +{{- end }} diff --git a/dist/chart/templates/rbac/manager-role.yaml b/dist/chart/templates/rbac/manager-role.yaml index a058eff..fb4c8e4 100644 --- a/dist/chart/templates/rbac/manager-role.yaml +++ b/dist/chart/templates/rbac/manager-role.yaml @@ -120,6 +120,7 @@ rules: resources: - temporalclusterclients - temporalclusterconnections + - temporalclusterproxies - temporalclusters - temporaldevservers - temporalnamespaces @@ -138,6 +139,7 @@ rules: resources: - temporalclusterclients/finalizers - temporalclusterconnections/finalizers + - temporalclusterproxies/finalizers - temporalclusters/finalizers - temporaldevservers/finalizers - temporalnamespaces/finalizers @@ -150,6 +152,7 @@ rules: resources: - temporalclusterclients/status - temporalclusterconnections/status + - temporalclusterproxies/status - temporalclusters/status - temporaldevservers/status - temporalnamespaces/status diff --git a/dist/chart/templates/webhook/mutating-webhook-configuration.yaml b/dist/chart/templates/webhook/mutating-webhook-configuration.yaml index a5b5cc7..6b03130 100644 --- a/dist/chart/templates/webhook/mutating-webhook-configuration.yaml +++ b/dist/chart/templates/webhook/mutating-webhook-configuration.yaml @@ -27,3 +27,23 @@ webhooks: resources: - temporalclusters sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: temporal-operator-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-temporal-bmor10-com-v1alpha1-temporalclusterproxy + failurePolicy: Fail + name: mtemporalclusterproxy-v1alpha1.kb.io + rules: + - apiGroups: + - temporal.bmor10.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - temporalclusterproxies + sideEffects: None diff --git a/dist/chart/templates/webhook/validating-webhook-configuration.yaml b/dist/chart/templates/webhook/validating-webhook-configuration.yaml index a4555ab..8e9f104 100644 --- a/dist/chart/templates/webhook/validating-webhook-configuration.yaml +++ b/dist/chart/templates/webhook/validating-webhook-configuration.yaml @@ -48,6 +48,26 @@ webhooks: resources: - temporalclusterconnections sideEffects: None + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: temporal-operator-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-temporal-bmor10-com-v1alpha1-temporalclusterproxy + failurePolicy: Fail + name: vtemporalclusterproxy-v1alpha1.kb.io + rules: + - apiGroups: + - temporal.bmor10.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - temporalclusterproxies + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/docs/api/v1alpha1.md b/docs/api/v1alpha1.md index 3262552..6ff8dbf 100644 --- a/docs/api/v1alpha1.md +++ b/docs/api/v1alpha1.md @@ -23,6 +23,7 @@ Package v1alpha1 contains API Schema definitions for the temporal v1alpha1 API g - [TemporalCluster](#temporalcluster) - [TemporalClusterClient](#temporalclusterclient) - [TemporalClusterConnection](#temporalclusterconnection) +- [TemporalClusterProxy](#temporalclusterproxy) - [TemporalDevServer](#temporaldevserver) - [TemporalNamespace](#temporalnamespace) - [TemporalSchedule](#temporalschedule) @@ -202,7 +203,9 @@ namespace: either a TemporalCluster (default) or a TemporalDevServer. _Appears in:_ - [ClusterConnectionPeer](#clusterconnectionpeer) +- [ProxyPeer](#proxypeer) - [TemporalClusterClientSpec](#temporalclusterclientspec) +- [TemporalClusterProxySpec](#temporalclusterproxyspec) - [TemporalNamespaceSpec](#temporalnamespacespec) - [TemporalScheduleSpec](#temporalschedulespec) - [TemporalSearchAttributeSpec](#temporalsearchattributespec) @@ -490,6 +493,7 @@ IssuerReference references a cert-manager Issuer or ClusterIssuer. _Appears in:_ - [MTLSSpec](#mtlsspec) +- [ProxyMuxTLS](#proxymuxtls) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -650,6 +654,200 @@ _Appears in:_ | `spec` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#rawextension-runtime-pkg)_ | Spec is a partial PodSpec (strategic-merge patch) merged onto the
generated pod template. It is stored as an opaque object to keep the
CRD schema small. | | Optional: \{\}
| +#### ProxyACL + + + +ProxyACL restricts what the proxy relays. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `allowedNamespaces` _string array_ | | | Optional: \{\}
| +| `allowedAdminMethods` _string array_ | AllowedAdminMethods defaults to the standard replication allowlist when empty. | | Optional: \{\}
| + + +#### ProxyFailoverVersionIncrement + + + +ProxyFailoverVersionIncrement translates failover-version increments across the link. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `local` _integer_ | | | | +| `remote` _integer_ | | | | + + +#### ProxyFieldMapping + + + +ProxyFieldMapping maps a local search-attribute field name to a remote one. + + + +_Appears in:_ +- [ProxySearchAttributeMapping](#proxysearchattributemapping) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `localFieldName` _string_ | | | | +| `remoteFieldName` _string_ | | | | + + +#### ProxyMux + + + +ProxyMux configures the s2s-proxy mux transport for one link. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `role` _string_ | Role selects whether this proxy opens a port (server) or dials out (client). | | Enum: [server client]
| +| `server` _[ProxyMuxServer](#proxymuxserver)_ | Server configures the listening side. Required when role=server. | | Optional: \{\}
| +| `client` _[ProxyMuxClient](#proxymuxclient)_ | Client configures the dialing side. Required when role=client. | | Optional: \{\}
| +| `muxCount` _integer_ | MuxCount is the number of multiplexed sessions. Defaults to the upstream default. | | Minimum: 1
Optional: \{\}
| +| `tls` _[ProxyMuxTLS](#proxymuxtls)_ | TLS configures the mux mTLS material. | | | + + +#### ProxyMuxClient + + + +ProxyMuxClient configures a mux-client (dialing) proxy. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `serverAddress` _string_ | ServerAddress is the reachable host:port of the remote mux-server. | | | + + +#### ProxyMuxServer + + + +ProxyMuxServer configures a mux-server (listening) proxy. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `listenPort` _integer_ | ListenPort is the port the mux listens on. | | Maximum: 65535
Minimum: 1
| +| `exposure` _[ServiceExposureSpec](#serviceexposurespec)_ | Exposure controls how the mux port is exposed (ClusterIP/NodePort/LoadBalancer). | | Optional: \{\}
| + + +#### ProxyMuxTLS + + + +ProxyMuxTLS configures the mux mTLS material for one side. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `provider` _string_ | Provider selects how this side's mux certificate is sourced. | cert-manager | Enum: [cert-manager secret]
Optional: \{\}
| +| `issuerRef` _[IssuerReference](#issuerreference)_ | IssuerRef mints this side's mux certificate. Required when provider=cert-manager. | | Optional: \{\}
| +| `secretRef` _[SecretReference](#secretreference)_ | SecretRef supplies BYO cert/key/CA. Required when provider=secret. | | Optional: \{\}
| +| `peerCARef` _[SecretReference](#secretreference)_ | PeerCARef supplies the remote side's CA to trust. When unset the CA bundle
from this side's own material is used (shared-issuer case). | | Optional: \{\}
| +| `caServerName` _string_ | CAServerName is the TLS server name a mux-client verifies against the
remote mux-server's certificate. s2s-proxy requires it on the client side
unless SkipCAVerification is true. When empty for a client, the operator
derives it from the host portion of mux.client.serverAddress (which matches
the server proxy's certificate SANs for operator-managed peers). Ignored
for the server role. | | Optional: \{\}
| +| `skipCAVerification` _boolean_ | SkipCAVerification disables CA verification of the mux peer's certificate.
Leave unset (false) for real mTLS; set true only for testing or when a
verifiable server name is unavailable. | | Optional: \{\}
| + + +#### ProxyNamespaceMapping + + + +ProxyNamespaceMapping maps a local namespace name to a remote one. + + + +_Appears in:_ +- [ProxyTranslation](#proxytranslation) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `local` _string_ | | | | +| `remote` _string_ | | | | + + +#### ProxyPeer + + + +ProxyPeer identifies the remote replication cluster reached over the mux. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the remote replication cluster name (== its currentClusterName). | | | +| `clusterRef` _[ClusterReference](#clusterreference)_ | ClusterRef optionally references an operator-managed remote TemporalCluster.
It is used only to reuse the peer's issuer CA when available. | | Optional: \{\}
| +| `enableConnection` _boolean_ | EnableConnection toggles replication without deleting the CR. | true | Optional: \{\}
| + + +#### ProxySearchAttributeMapping + + + +ProxySearchAttributeMapping maps search-attribute field names for a namespace. + + + +_Appears in:_ +- [ProxyTranslation](#proxytranslation) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespace` _string_ | | | | +| `mappings` _[ProxyFieldMapping](#proxyfieldmapping) array_ | | | | + + +#### ProxyTranslation + + + +ProxyTranslation renames namespaces and search attributes in-flight. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespaces` _[ProxyNamespaceMapping](#proxynamespacemapping) array_ | | | Optional: \{\}
| +| `searchAttributes` _[ProxySearchAttributeMapping](#proxysearchattributemapping) array_ | | | Optional: \{\}
| + + #### RetryPolicySpec @@ -843,6 +1041,7 @@ for connecting to an external Temporal peer. Keys default to the conventional _Appears in:_ - [ClusterConnectionPeer](#clusterconnectionpeer) +- [ProxyMuxTLS](#proxymuxtls) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -861,6 +1060,7 @@ ServiceExposureSpec configures how a service is exposed. _Appears in:_ +- [ProxyMuxServer](#proxymuxserver) - [ServiceSpec](#servicespec) - [TemporalDevServerSpec](#temporaldevserverspec) @@ -1107,6 +1307,50 @@ _Appears in:_ +#### TemporalClusterProxy + + + +TemporalClusterProxy is the Schema for the temporalclusterproxies API. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `temporal.bmor10.com/v1alpha1` | | | +| `kind` _string_ | `TemporalClusterProxy` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[TemporalClusterProxySpec](#temporalclusterproxyspec)_ | | | Required: \{\}
| + + +#### TemporalClusterProxySpec + + + +TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link +to one replication peer over an s2s-proxy mux connection. + + + +_Appears in:_ +- [TemporalClusterProxy](#temporalclusterproxy) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `localClusterRef` _[ClusterReference](#clusterreference)_ | LocalClusterRef references the local operator-managed TemporalCluster this
proxy fronts. Its frontend address and issuer CA are resolved automatically. | | | +| `localClusterName` _string_ | LocalClusterName overrides the replication-group name of the local cluster.
Defaults to the referenced cluster's clusterMetadata.currentClusterName. | | Optional: \{\}
| +| `peer` _[ProxyPeer](#proxypeer)_ | Peer is the remote replication cluster on the far side of the mux. | | | +| `mux` _[ProxyMux](#proxymux)_ | Mux configures the s2s-proxy multiplexed transport. | | | +| `translation` _[ProxyTranslation](#proxytranslation)_ | Translation optionally renames namespaces and search attributes in-flight. | | Optional: \{\}
| +| `failoverVersionIncrement` _[ProxyFailoverVersionIncrement](#proxyfailoverversionincrement)_ | FailoverVersionIncrement optionally translates failover-version increments. | | Optional: \{\}
| +| `acl` _[ProxyACL](#proxyacl)_ | ACL optionally restricts the admin methods and namespaces the proxy relays. | | Optional: \{\}
| +| `image` _string_ | Image overrides the pinned s2s-proxy image. | | Optional: \{\}
| + + + + #### TemporalClusterSpec diff --git a/docs/content/reference/_index.md b/docs/content/reference/_index.md index 4ab699c..d7e0ee1 100644 --- a/docs/content/reference/_index.md +++ b/docs/content/reference/_index.md @@ -28,6 +28,7 @@ Package v1alpha1 contains API Schema definitions for the temporal v1alpha1 API g - [TemporalCluster](#temporalcluster) - [TemporalClusterClient](#temporalclusterclient) - [TemporalClusterConnection](#temporalclusterconnection) +- [TemporalClusterProxy](#temporalclusterproxy) - [TemporalDevServer](#temporaldevserver) - [TemporalNamespace](#temporalnamespace) - [TemporalSchedule](#temporalschedule) @@ -207,7 +208,9 @@ namespace: either a TemporalCluster (default) or a TemporalDevServer. _Appears in:_ - [ClusterConnectionPeer](#clusterconnectionpeer) +- [ProxyPeer](#proxypeer) - [TemporalClusterClientSpec](#temporalclusterclientspec) +- [TemporalClusterProxySpec](#temporalclusterproxyspec) - [TemporalNamespaceSpec](#temporalnamespacespec) - [TemporalScheduleSpec](#temporalschedulespec) - [TemporalSearchAttributeSpec](#temporalsearchattributespec) @@ -495,6 +498,7 @@ IssuerReference references a cert-manager Issuer or ClusterIssuer. _Appears in:_ - [MTLSSpec](#mtlsspec) +- [ProxyMuxTLS](#proxymuxtls) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -655,6 +659,200 @@ _Appears in:_ | `spec` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#rawextension-runtime-pkg)_ | Spec is a partial PodSpec (strategic-merge patch) merged onto the
generated pod template. It is stored as an opaque object to keep the
CRD schema small. | | Optional: \{\}
| +#### ProxyACL + + + +ProxyACL restricts what the proxy relays. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `allowedNamespaces` _string array_ | | | Optional: \{\}
| +| `allowedAdminMethods` _string array_ | AllowedAdminMethods defaults to the standard replication allowlist when empty. | | Optional: \{\}
| + + +#### ProxyFailoverVersionIncrement + + + +ProxyFailoverVersionIncrement translates failover-version increments across the link. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `local` _integer_ | | | | +| `remote` _integer_ | | | | + + +#### ProxyFieldMapping + + + +ProxyFieldMapping maps a local search-attribute field name to a remote one. + + + +_Appears in:_ +- [ProxySearchAttributeMapping](#proxysearchattributemapping) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `localFieldName` _string_ | | | | +| `remoteFieldName` _string_ | | | | + + +#### ProxyMux + + + +ProxyMux configures the s2s-proxy mux transport for one link. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `role` _string_ | Role selects whether this proxy opens a port (server) or dials out (client). | | Enum: [server client]
| +| `server` _[ProxyMuxServer](#proxymuxserver)_ | Server configures the listening side. Required when role=server. | | Optional: \{\}
| +| `client` _[ProxyMuxClient](#proxymuxclient)_ | Client configures the dialing side. Required when role=client. | | Optional: \{\}
| +| `muxCount` _integer_ | MuxCount is the number of multiplexed sessions. Defaults to the upstream default. | | Minimum: 1
Optional: \{\}
| +| `tls` _[ProxyMuxTLS](#proxymuxtls)_ | TLS configures the mux mTLS material. | | | + + +#### ProxyMuxClient + + + +ProxyMuxClient configures a mux-client (dialing) proxy. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `serverAddress` _string_ | ServerAddress is the reachable host:port of the remote mux-server. | | | + + +#### ProxyMuxServer + + + +ProxyMuxServer configures a mux-server (listening) proxy. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `listenPort` _integer_ | ListenPort is the port the mux listens on. | | Maximum: 65535
Minimum: 1
| +| `exposure` _[ServiceExposureSpec](#serviceexposurespec)_ | Exposure controls how the mux port is exposed (ClusterIP/NodePort/LoadBalancer). | | Optional: \{\}
| + + +#### ProxyMuxTLS + + + +ProxyMuxTLS configures the mux mTLS material for one side. + + + +_Appears in:_ +- [ProxyMux](#proxymux) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `provider` _string_ | Provider selects how this side's mux certificate is sourced. | cert-manager | Enum: [cert-manager secret]
Optional: \{\}
| +| `issuerRef` _[IssuerReference](#issuerreference)_ | IssuerRef mints this side's mux certificate. Required when provider=cert-manager. | | Optional: \{\}
| +| `secretRef` _[SecretReference](#secretreference)_ | SecretRef supplies BYO cert/key/CA. Required when provider=secret. | | Optional: \{\}
| +| `peerCARef` _[SecretReference](#secretreference)_ | PeerCARef supplies the remote side's CA to trust. When unset the CA bundle
from this side's own material is used (shared-issuer case). | | Optional: \{\}
| +| `caServerName` _string_ | CAServerName is the TLS server name a mux-client verifies against the
remote mux-server's certificate. s2s-proxy requires it on the client side
unless SkipCAVerification is true. When empty for a client, the operator
derives it from the host portion of mux.client.serverAddress (which matches
the server proxy's certificate SANs for operator-managed peers). Ignored
for the server role. | | Optional: \{\}
| +| `skipCAVerification` _boolean_ | SkipCAVerification disables CA verification of the mux peer's certificate.
Leave unset (false) for real mTLS; set true only for testing or when a
verifiable server name is unavailable. | | Optional: \{\}
| + + +#### ProxyNamespaceMapping + + + +ProxyNamespaceMapping maps a local namespace name to a remote one. + + + +_Appears in:_ +- [ProxyTranslation](#proxytranslation) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `local` _string_ | | | | +| `remote` _string_ | | | | + + +#### ProxyPeer + + + +ProxyPeer identifies the remote replication cluster reached over the mux. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the remote replication cluster name (== its currentClusterName). | | | +| `clusterRef` _[ClusterReference](#clusterreference)_ | ClusterRef optionally references an operator-managed remote TemporalCluster.
It is used only to reuse the peer's issuer CA when available. | | Optional: \{\}
| +| `enableConnection` _boolean_ | EnableConnection toggles replication without deleting the CR. | true | Optional: \{\}
| + + +#### ProxySearchAttributeMapping + + + +ProxySearchAttributeMapping maps search-attribute field names for a namespace. + + + +_Appears in:_ +- [ProxyTranslation](#proxytranslation) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespace` _string_ | | | | +| `mappings` _[ProxyFieldMapping](#proxyfieldmapping) array_ | | | | + + +#### ProxyTranslation + + + +ProxyTranslation renames namespaces and search attributes in-flight. + + + +_Appears in:_ +- [TemporalClusterProxySpec](#temporalclusterproxyspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespaces` _[ProxyNamespaceMapping](#proxynamespacemapping) array_ | | | Optional: \{\}
| +| `searchAttributes` _[ProxySearchAttributeMapping](#proxysearchattributemapping) array_ | | | Optional: \{\}
| + + #### RetryPolicySpec @@ -848,6 +1046,7 @@ for connecting to an external Temporal peer. Keys default to the conventional _Appears in:_ - [ClusterConnectionPeer](#clusterconnectionpeer) +- [ProxyMuxTLS](#proxymuxtls) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -866,6 +1065,7 @@ ServiceExposureSpec configures how a service is exposed. _Appears in:_ +- [ProxyMuxServer](#proxymuxserver) - [ServiceSpec](#servicespec) - [TemporalDevServerSpec](#temporaldevserverspec) @@ -1112,6 +1312,50 @@ _Appears in:_ +#### TemporalClusterProxy + + + +TemporalClusterProxy is the Schema for the temporalclusterproxies API. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `temporal.bmor10.com/v1alpha1` | | | +| `kind` _string_ | `TemporalClusterProxy` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[TemporalClusterProxySpec](#temporalclusterproxyspec)_ | | | Required: \{\}
| + + +#### TemporalClusterProxySpec + + + +TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link +to one replication peer over an s2s-proxy mux connection. + + + +_Appears in:_ +- [TemporalClusterProxy](#temporalclusterproxy) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `localClusterRef` _[ClusterReference](#clusterreference)_ | LocalClusterRef references the local operator-managed TemporalCluster this
proxy fronts. Its frontend address and issuer CA are resolved automatically. | | | +| `localClusterName` _string_ | LocalClusterName overrides the replication-group name of the local cluster.
Defaults to the referenced cluster's clusterMetadata.currentClusterName. | | Optional: \{\}
| +| `peer` _[ProxyPeer](#proxypeer)_ | Peer is the remote replication cluster on the far side of the mux. | | | +| `mux` _[ProxyMux](#proxymux)_ | Mux configures the s2s-proxy multiplexed transport. | | | +| `translation` _[ProxyTranslation](#proxytranslation)_ | Translation optionally renames namespaces and search attributes in-flight. | | Optional: \{\}
| +| `failoverVersionIncrement` _[ProxyFailoverVersionIncrement](#proxyfailoverversionincrement)_ | FailoverVersionIncrement optionally translates failover-version increments. | | Optional: \{\}
| +| `acl` _[ProxyACL](#proxyacl)_ | ACL optionally restricts the admin methods and namespaces the proxy relays. | | Optional: \{\}
| +| `image` _string_ | Image overrides the pinned s2s-proxy image. | | Optional: \{\}
| + + + + #### TemporalClusterSpec diff --git a/docs/superpowers/plans/2026-07-03-temporalclusterproxy-s2s-proxy.md b/docs/superpowers/plans/2026-07-03-temporalclusterproxy-s2s-proxy.md new file mode 100644 index 0000000..f0e05dd --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-temporalclusterproxy-s2s-proxy.md @@ -0,0 +1,1618 @@ +# TemporalClusterProxy (s2s-proxy automation) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `TemporalClusterProxy` CRD that deploys and wires `temporalio/s2s-proxy` so operator-managed Temporal clusters replicate across segregated networks over mux + mTLS. + +**Architecture:** A new namespaced CRD whose reconciler renders an s2s-proxy `Deployment` + `Service` + `ConfigMap` (+ optional cert-manager `Certificate`), all owned by the CR, and then registers the peer as a remote cluster on the *local* Temporal via the *local* proxy's `tcpServer` address (reusing the existing `internal/temporal.RemoteClusterClient`). Resource builders and config rendering are pure (no client/IO) so they stay `wasm`-compatible, matching the existing `internal/resources` convention. + +**Tech Stack:** Go 1.26.4, kubebuilder/controller-runtime v0.23 (typed generic admission API), cert-manager, `sigs.k8s.io/yaml` for config marshalling, envtest + chainsaw for tests. + +## Global Constraints + +- Go version: `1.26.4` in `go.mod`; CI uses `1.26.x`. (verbatim from spec/memories) +- Module path: `github.com/bmorton/temporal-operator`; API group: `temporal.bmor10.com`; copyright owner "Brian Morton" (Apache-2.0 header on every new `.go` file — copy verbatim from any existing file in the same package). +- Resource builders in `internal/resources` and config rendering must be pure (no k8s client / IO) so `GOOS=js GOARCH=wasm go build ./internal/resources/ ./internal/temporal/ ./api/...` stays green. +- Webhooks use the typed generic admission API: `admission.Validator[*T]` / `admission.Defaulter[*T]`, registered with `ctrl.NewWebhookManagedBy(mgr, &T{}).WithValidator(...)`. The `WithCustomValidator`/`.For()` variants are removed. +- After changing `api/v1alpha1`: run `make generate manifests`, then `make api-docs docs-crd-reference` and commit `docs/api/v1alpha1.md` + `docs/content/reference/_index.md` (docs CI drift check). +- After changing API types or RBAC markers: run `make helm-chart` and commit `dist/chart` (verify-chart CI). Do NOT hand-edit `dist/chart`. Do NOT commit `.github/workflows/test-chart.yml` (kept deleted/untracked). +- Every commit: Conventional Commit prefix (`feat`/`test`/`docs`/`chore`) + DCO sign-off (`git commit -s`) + the `Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>` trailer. +- Pin the s2s-proxy image (binary-only project; internal APIs may change). Verified pinned image: `temporalio/s2s-proxy:v0.2.1` (latest published tag as of 2026-07). The container entrypoint runs `s2s-proxy start --config $CONFIG_YML`, so the config path is passed via the `CONFIG_YML` env var (verified from the upstream `scripts/start.sh`). + +--- + +## File structure + +**Create:** +- `api/v1alpha1/temporalclusterproxy_types.go` — CRD Go types + `init()` registration. +- `internal/resources/clusterproxyconfig.go` — pure s2s-proxy config-YAML render. +- `internal/resources/clusterproxy.go` — pure `Build*` funcs (ConfigMap, Certificate, Deployment, Service) + name helpers. +- `internal/resources/clusterproxy_test.go` — unit tests for the two files above. +- `internal/controller/temporalclusterproxy_controller.go` — the reconciler. +- `internal/controller/temporalclusterproxy_controller_test.go` — envtest suite. +- `internal/webhook/v1alpha1/temporalclusterproxy_webhook.go` — validator + defaulter. +- `internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go` — webhook tests. +- `examples/cluster-proxy-mux/server.yaml`, `examples/cluster-proxy-mux/client.yaml` — example CRs. +- `test/e2e/clusterproxy/chainsaw-test.yaml` (+ manifests) — e2e (final task). + +**Modify:** +- `cmd/main.go` — register the reconciler + webhook. +- Generated (via make targets, do not hand-edit): `api/v1alpha1/zz_generated.deepcopy.go`, `config/crd/...`, `config/rbac/role.yaml`, `dist/chart/**`, `docs/api/v1alpha1.md`, `docs/content/reference/_index.md`. + +--- + +## Task 1: API types + +**Files:** +- Create: `api/v1alpha1/temporalclusterproxy_types.go` +- Modify (generated): `api/v1alpha1/zz_generated.deepcopy.go`, `config/crd/`, `dist/chart/`, docs + +**Interfaces:** +- Produces (used by every later task): + - `type TemporalClusterProxy struct{ ...; Spec TemporalClusterProxySpec; Status TemporalClusterProxyStatus }` + - `TemporalClusterProxySpec{ LocalClusterRef ClusterReference; LocalClusterName string; Peer ProxyPeer; Mux ProxyMux; Translation *ProxyTranslation; FailoverVersionIncrement *ProxyFailoverVersionIncrement; ACL *ProxyACL; Image string }` + - `ProxyPeer{ Name string; ClusterRef *ClusterReference; EnableConnection *bool }` + - `ProxyMux{ Role string; Server *ProxyMuxServer; Client *ProxyMuxClient; MuxCount *int32; TLS ProxyMuxTLS }` + - `ProxyMuxServer{ ListenPort int32; Exposure *ServiceExposureSpec }` + - `ProxyMuxClient{ ServerAddress string }` + - `ProxyMuxTLS{ Provider string; IssuerRef *IssuerReference; SecretRef *SecretReference; PeerCARef *SecretReference }` + - `ProxyTranslation{ Namespaces []ProxyNamespaceMapping; SearchAttributes []ProxySearchAttributeMapping }` + - `ProxyNamespaceMapping{ Local, Remote string }` + - `ProxySearchAttributeMapping{ Namespace string; Mappings []ProxyFieldMapping }` + - `ProxyFieldMapping{ LocalFieldName, RemoteFieldName string }` + - `ProxyFailoverVersionIncrement{ Local, Remote int64 }` + - `ProxyACL{ AllowedNamespaces, AllowedAdminMethods []string }` + - `TemporalClusterProxyStatus{ ObservedGeneration int64; ProxyEndpoint string; Conditions []metav1.Condition }` + - constants: `ProxyRoleServer = "server"`, `ProxyRoleClient = "client"`, `ConditionProxyDeployed = "ProxyDeployed"`, `ConditionRemoteClusterRegistered = "RemoteClusterRegistered"`. + +- [ ] **Step 1: Write the type file** + +Create `api/v1alpha1/temporalclusterproxy_types.go` (copy the Apache header verbatim from `temporalclusterconnection_types.go`): + +```go +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Mux roles for ProxyMux.Role. +const ( + // ProxyRoleServer opens a listening mux port (the exposed side). + ProxyRoleServer = "server" + // ProxyRoleClient dials out to a remote mux-server (never opens a port). + ProxyRoleClient = "client" +) + +// TemporalClusterProxySpec describes one local cluster's s2s-proxy and its link +// to one replication peer over an s2s-proxy mux connection. +type TemporalClusterProxySpec struct { + // LocalClusterRef references the local operator-managed TemporalCluster this + // proxy fronts. Its frontend address and issuer CA are resolved automatically. + LocalClusterRef ClusterReference `json:"localClusterRef"` + + // LocalClusterName overrides the replication-group name of the local cluster. + // Defaults to the referenced cluster's clusterMetadata.currentClusterName. + // +optional + LocalClusterName string `json:"localClusterName,omitempty"` + + // Peer is the remote replication cluster on the far side of the mux. + Peer ProxyPeer `json:"peer"` + + // Mux configures the s2s-proxy multiplexed transport. + Mux ProxyMux `json:"mux"` + + // Translation optionally renames namespaces and search attributes in-flight. + // +optional + Translation *ProxyTranslation `json:"translation,omitempty"` + + // FailoverVersionIncrement optionally translates failover-version increments. + // +optional + FailoverVersionIncrement *ProxyFailoverVersionIncrement `json:"failoverVersionIncrement,omitempty"` + + // ACL optionally restricts the admin methods and namespaces the proxy relays. + // +optional + ACL *ProxyACL `json:"acl,omitempty"` + + // Image overrides the pinned s2s-proxy image. + // +optional + Image string `json:"image,omitempty"` +} + +// ProxyPeer identifies the remote replication cluster reached over the mux. +type ProxyPeer struct { + // Name is the remote replication cluster name (== its currentClusterName). + Name string `json:"name"` + + // ClusterRef optionally references an operator-managed remote TemporalCluster. + // It is used only to reuse the peer's issuer CA when available. + // +optional + ClusterRef *ClusterReference `json:"clusterRef,omitempty"` + + // EnableConnection toggles replication without deleting the CR. + // +kubebuilder:default=true + // +optional + EnableConnection *bool `json:"enableConnection,omitempty"` +} + +// ProxyMux configures the s2s-proxy mux transport for one link. +type ProxyMux struct { + // Role selects whether this proxy opens a port (server) or dials out (client). + // +kubebuilder:validation:Enum=server;client + Role string `json:"role"` + + // Server configures the listening side. Required when role=server. + // +optional + Server *ProxyMuxServer `json:"server,omitempty"` + + // Client configures the dialing side. Required when role=client. + // +optional + Client *ProxyMuxClient `json:"client,omitempty"` + + // MuxCount is the number of multiplexed sessions. Defaults to the upstream default. + // +kubebuilder:validation:Minimum=1 + // +optional + MuxCount *int32 `json:"muxCount,omitempty"` + + // TLS configures the mux mTLS material. + TLS ProxyMuxTLS `json:"tls"` +} + +// ProxyMuxServer configures a mux-server (listening) proxy. +type ProxyMuxServer struct { + // ListenPort is the port the mux listens on. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + ListenPort int32 `json:"listenPort"` + + // Exposure controls how the mux port is exposed (ClusterIP/NodePort/LoadBalancer). + // +optional + Exposure *ServiceExposureSpec `json:"exposure,omitempty"` +} + +// ProxyMuxClient configures a mux-client (dialing) proxy. +type ProxyMuxClient struct { + // ServerAddress is the reachable host:port of the remote mux-server. + ServerAddress string `json:"serverAddress"` +} + +// ProxyMuxTLS configures the mux mTLS material for one side. +type ProxyMuxTLS struct { + // Provider selects how this side's mux certificate is sourced. + // +kubebuilder:validation:Enum=cert-manager;secret + // +kubebuilder:default=cert-manager + // +optional + Provider string `json:"provider,omitempty"` + + // IssuerRef mints this side's mux certificate. Required when provider=cert-manager. + // +optional + IssuerRef *IssuerReference `json:"issuerRef,omitempty"` + + // SecretRef supplies BYO cert/key/CA. Required when provider=secret. + // +optional + SecretRef *SecretReference `json:"secretRef,omitempty"` + + // PeerCARef supplies the remote side's CA to trust. When unset the CA bundle + // from this side's own material is used (shared-issuer case). + // +optional + PeerCARef *SecretReference `json:"peerCARef,omitempty"` +} + +// ProxyTranslation renames namespaces and search attributes in-flight. +type ProxyTranslation struct { + // +optional + Namespaces []ProxyNamespaceMapping `json:"namespaces,omitempty"` + // +optional + SearchAttributes []ProxySearchAttributeMapping `json:"searchAttributes,omitempty"` +} + +// ProxyNamespaceMapping maps a local namespace name to a remote one. +type ProxyNamespaceMapping struct { + Local string `json:"local"` + Remote string `json:"remote"` +} + +// ProxySearchAttributeMapping maps search-attribute field names for a namespace. +type ProxySearchAttributeMapping struct { + Namespace string `json:"namespace"` + Mappings []ProxyFieldMapping `json:"mappings"` +} + +// ProxyFieldMapping maps a local search-attribute field name to a remote one. +type ProxyFieldMapping struct { + LocalFieldName string `json:"localFieldName"` + RemoteFieldName string `json:"remoteFieldName"` +} + +// ProxyFailoverVersionIncrement translates failover-version increments across the link. +type ProxyFailoverVersionIncrement struct { + Local int64 `json:"local"` + Remote int64 `json:"remote"` +} + +// ProxyACL restricts what the proxy relays. +type ProxyACL struct { + // +optional + AllowedNamespaces []string `json:"allowedNamespaces,omitempty"` + // AllowedAdminMethods defaults to the standard replication allowlist when empty. + // +optional + AllowedAdminMethods []string `json:"allowedAdminMethods,omitempty"` +} + +// TemporalClusterProxyStatus is the observed state. +type TemporalClusterProxyStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // ProxyEndpoint reports the exposed mux address (server role) to hand to the peer. + // +optional + ProxyEndpoint string `json:"proxyEndpoint,omitempty"` + + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=tcproxy +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Role",type=string,JSONPath=`.spec.mux.role` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// TemporalClusterProxy is the Schema for the temporalclusterproxies API. +type TemporalClusterProxy struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + // +required + Spec TemporalClusterProxySpec `json:"spec"` + // +optional + Status TemporalClusterProxyStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// TemporalClusterProxyList contains a list of TemporalClusterProxy. +type TemporalClusterProxyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []TemporalClusterProxy `json:"items"` +} + +func init() { + registerType(&TemporalClusterProxy{}, &TemporalClusterProxyList{}) +} +``` + +- [ ] **Step 2: Add condition + reason constants** + +In `api/v1alpha1/conditions.go`, add next to the existing condition/reason constants: + +```go +// ConditionProxyDeployed indicates the s2s-proxy Deployment is available. +ConditionProxyDeployed = "ProxyDeployed" +// ConditionRemoteClusterRegistered indicates the local Temporal registered the +// peer as a remote cluster via the local proxy. +ConditionRemoteClusterRegistered = "RemoteClusterRegistered" +``` + +And add reason constants (match the existing `Reason*` style in that file): + +```go +ReasonProxyNotReady = "ProxyNotReady" +ReasonProxyReady = "ProxyReady" +ReasonClusterNotReady = "ClusterNotReady" +ReasonMTLSNotReady = "MTLSNotReady" +ReasonRegistrationFailed = "RegistrationFailed" +``` + +(If any of these names already exist in `conditions.go`, reuse the existing one and drop the duplicate.) + +- [ ] **Step 3: Generate deepcopy + manifests** + +Run: `make generate manifests` +Expected: `api/v1alpha1/zz_generated.deepcopy.go` gains `DeepCopy*` for the new types; a new `config/crd/bases/temporal.bmor10.com_temporalclusterproxies.yaml` appears; `config/rbac/role.yaml` unchanged (RBAC comes in Task 4). No errors. + +- [ ] **Step 4: Verify it compiles + wasm build** + +Run: `go build ./... && GOOS=js GOARCH=wasm go build ./internal/resources/ ./internal/temporal/ ./api/...` +Expected: both succeed. + +- [ ] **Step 5: Regenerate docs + chart** + +Run: `make api-docs docs-crd-reference helm-chart` +Then: `git status` — expect changes under `docs/api/v1alpha1.md`, `docs/content/reference/_index.md`, `dist/chart/`, `config/`. If `.github/workflows/test-chart.yml` reappears, `rm` it (do not stage). + +- [ ] **Step 6: Commit** + +```bash +git add api/ config/ dist/chart/ docs/api docs/content +git commit -s -m "feat(api): add TemporalClusterProxy CRD types + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 2: s2s-proxy config rendering (pure) + +**Files:** +- Create: `internal/resources/clusterproxyconfig.go` +- Test: `internal/resources/clusterproxy_test.go` + +**Interfaces:** +- Consumes: `TemporalClusterProxy` types (Task 1). +- Produces: + - `func BuildClusterProxyConfig(cr *temporalv1alpha1.TemporalClusterProxy, localFrontendAddress string) (string, error)` — returns the s2s-proxy config YAML. + - Path constants: `ProxyTLSMountPath = "/etc/s2s-proxy/tls"`, `ProxyPeerCAMountPath = "/etc/s2s-proxy/peer-ca"`, `ProxyConfigMountPath = "/etc/s2s-proxy"`, `ProxyConfigFileName = "config.yaml"`. + - Port constant: `ProxyTCPServerPort int32 = 6233`. + - `func DefaultAllowedAdminMethods() []string` — the standard replication allowlist. + +- [ ] **Step 1: Write the failing test** + +In `internal/resources/clusterproxy_test.go` (copy the Apache header from `builders_test.go`): + +```go +package resources_test + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + sigsyaml "sigs.k8s.io/yaml" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" +) + +func serverProxyCR() *temporalv1alpha1.TemporalClusterProxy { + enable := true + return &temporalv1alpha1.TemporalClusterProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "link", Namespace: "temporal-system"}, + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: "cluster-a"}, + LocalClusterName: "cluster-a", + Peer: temporalv1alpha1.ProxyPeer{Name: "cluster-b", EnableConnection: &enable}, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ListenPort: 6334}, + TLS: temporalv1alpha1.ProxyMuxTLS{Provider: "cert-manager"}, + }, + }, + } +} + +func TestBuildClusterProxyConfig_ServerRole(t *testing.T) { + out, err := resources.BuildClusterProxyConfig(serverProxyCR(), "cluster-a-frontend.temporal-system.svc.cluster.local:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + + var cfg struct { + ClusterConnections []struct { + Name string `json:"name"` + Local struct { + ConnectionType string `json:"connectionType"` + TCPClient struct{ Address string } `json:"tcpClient"` + TCPServer struct{ Address string } `json:"tcpServer"` + } `json:"local"` + Remote struct { + ConnectionType string `json:"connectionType"` + MuxAddressInfo struct { + Address string `json:"address"` + TLS struct { + CertificatePath string `json:"certificatePath"` + KeyPath string `json:"keyPath"` + RemoteCAPath string `json:"remoteCAPath"` + } `json:"tls"` + } `json:"muxAddressInfo"` + } `json:"remote"` + } `json:"clusterConnections"` + } + if err := sigsyaml.Unmarshal([]byte(out), &cfg); err != nil { + t.Fatalf("unmarshal rendered config: %v\n%s", err, out) + } + if len(cfg.ClusterConnections) != 1 { + t.Fatalf("want 1 connection, got %d", len(cfg.ClusterConnections)) + } + c := cfg.ClusterConnections[0] + if c.Local.ConnectionType != "tcp" { + t.Errorf("local.connectionType = %q, want tcp", c.Local.ConnectionType) + } + if c.Local.TCPClient.Address != "cluster-a-frontend.temporal-system.svc.cluster.local:7233" { + t.Errorf("tcpClient.address = %q", c.Local.TCPClient.Address) + } + if !strings.HasSuffix(c.Local.TCPServer.Address, "6233") { + t.Errorf("tcpServer.address = %q, want :6233", c.Local.TCPServer.Address) + } + if c.Remote.ConnectionType != "mux-server" { + t.Errorf("remote.connectionType = %q, want mux-server", c.Remote.ConnectionType) + } + if !strings.HasSuffix(c.Remote.MuxAddressInfo.Address, "6334") { + t.Errorf("mux address = %q, want :6334", c.Remote.MuxAddressInfo.Address) + } + if c.Remote.MuxAddressInfo.TLS.CertificatePath != resources.ProxyTLSMountPath+"/tls.crt" { + t.Errorf("certificatePath = %q", c.Remote.MuxAddressInfo.TLS.CertificatePath) + } + if c.Remote.MuxAddressInfo.TLS.RemoteCAPath != resources.ProxyTLSMountPath+"/ca.crt" { + t.Errorf("remoteCAPath = %q (want own ca.crt when no peerCARef)", c.Remote.MuxAddressInfo.TLS.RemoteCAPath) + } +} + +func TestBuildClusterProxyConfig_ClientRoleWithTranslation(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + cr.Spec.Mux.Server = nil + cr.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "b.example.com:6334"} + cr.Spec.Translation = &temporalv1alpha1.ProxyTranslation{ + Namespaces: []temporalv1alpha1.ProxyNamespaceMapping{{Local: "ns", Remote: "ns.acct"}}, + } + + out, err := resources.BuildClusterProxyConfig(cr, "cluster-a-frontend:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(out, "mux-client") { + t.Errorf("expected mux-client in:\n%s", out) + } + if !strings.Contains(out, "b.example.com:6334") { + t.Errorf("expected serverAddress in:\n%s", out) + } + if !strings.Contains(out, "ns.acct") { + t.Errorf("expected namespace translation in:\n%s", out) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `go test ./internal/resources/ -run TestBuildClusterProxyConfig -v` +Expected: FAIL — `undefined: resources.BuildClusterProxyConfig`. + +- [ ] **Step 3: Write the implementation** + +Create `internal/resources/clusterproxyconfig.go` (Apache header): + +```go +package resources + +import ( + "fmt" + + sigsyaml "sigs.k8s.io/yaml" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +// Mount paths and ports for the rendered s2s-proxy pod. +const ( + ProxyConfigMountPath = "/etc/s2s-proxy" + ProxyConfigFileName = "config.yaml" + ProxyTLSMountPath = "/etc/s2s-proxy/tls" + ProxyPeerCAMountPath = "/etc/s2s-proxy/peer-ca" + + // ProxyTCPServerPort is the port the proxy exposes for the local Temporal to + // reach as the peer's frontend address. + ProxyTCPServerPort int32 = 6233 +) + +// DefaultAllowedAdminMethods is the standard admin-service allowlist required +// for cross-cluster replication. +func DefaultAllowedAdminMethods() []string { + return []string{ + "AddOrUpdateRemoteCluster", + "DescribeCluster", + "DescribeMutableState", + "GetNamespaceReplicationMessages", + "GetWorkflowExecutionRawHistoryV2", + "ListClusters", + "StreamWorkflowReplicationMessages", + } +} + +// --- s2s-proxy config schema (subset we render) --- + +type proxyTLS struct { + CertificatePath string `json:"certificatePath"` + KeyPath string `json:"keyPath"` + RemoteCAPath string `json:"remoteCAPath"` +} + +type proxyAddressInfo struct { + Address string `json:"address"` + TLS proxyTLS `json:"tls"` +} + +type proxyTCPEndpoint struct { + Address string `json:"address"` +} + +type proxyLocal struct { + ConnectionType string `json:"connectionType"` + TCPClient proxyTCPEndpoint `json:"tcpClient"` + TCPServer proxyTCPEndpoint `json:"tcpServer"` +} + +type proxyRemote struct { + ConnectionType string `json:"connectionType"` + MuxCount *int32 `json:"muxCount,omitempty"` + MuxAddressInfo proxyAddressInfo `json:"muxAddressInfo"` +} + +type proxyNamespaceMapping struct { + Local string `json:"local"` + Remote string `json:"remote"` +} + +type proxyNamespaceTranslation struct { + Mappings []proxyNamespaceMapping `json:"mappings"` +} + +type proxyFieldMapping struct { + LocalFieldName string `json:"localFieldName"` + RemoteFieldName string `json:"remoteFieldName"` +} + +type proxySANamespaceMapping struct { + Name string `json:"name"` + NamespaceID string `json:"namespaceId"` + Mappings []proxyFieldMapping `json:"mappings"` +} + +type proxySATranslation struct { + NamespaceMappings []proxySANamespaceMapping `json:"namespaceMappings"` +} + +type proxyFailover struct { + Local int64 `json:"local"` + Remote int64 `json:"remote"` +} + +type proxyACLPolicy struct { + AllowedMethods map[string][]string `json:"allowedMethods"` + AllowedNamespaces []string `json:"allowedNamespaces,omitempty"` +} + +type proxyConnection struct { + Name string `json:"name"` + Local proxyLocal `json:"local"` + Remote proxyRemote `json:"remote"` + NamespaceTranslation *proxyNamespaceTranslation `json:"namespaceTranslation,omitempty"` + SearchAttributeTranslation *proxySATranslation `json:"searchAttributeTranslation,omitempty"` + FailoverVersionIncrementTranslation *proxyFailover `json:"failoverVersionIncrementTranslation,omitempty"` + ACLPolicy *proxyACLPolicy `json:"aclPolicy,omitempty"` +} + +type proxyConfigFile struct { + ClusterConnections []proxyConnection `json:"clusterConnections"` +} + +// BuildClusterProxyConfig renders the s2s-proxy config YAML for one CR. It is +// pure: localFrontendAddress is resolved by the caller. +func BuildClusterProxyConfig(cr *temporalv1alpha1.TemporalClusterProxy, localFrontendAddress string) (string, error) { + mux := cr.Spec.Mux + + remote := proxyRemote{ + MuxCount: mux.MuxCount, + MuxAddressInfo: proxyAddressInfo{ + TLS: proxyTLS{ + CertificatePath: ProxyTLSMountPath + "/tls.crt", + KeyPath: ProxyTLSMountPath + "/tls.key", + RemoteCAPath: proxyRemoteCAPath(cr), + }, + }, + } + switch mux.Role { + case temporalv1alpha1.ProxyRoleServer: + if mux.Server == nil { + return "", fmt.Errorf("mux.server is required for role=server") + } + remote.ConnectionType = "mux-server" + remote.MuxAddressInfo.Address = fmt.Sprintf("0.0.0.0:%d", mux.Server.ListenPort) + case temporalv1alpha1.ProxyRoleClient: + if mux.Client == nil { + return "", fmt.Errorf("mux.client is required for role=client") + } + remote.ConnectionType = "mux-client" + remote.MuxAddressInfo.Address = mux.Client.ServerAddress + default: + return "", fmt.Errorf("unknown mux.role %q", mux.Role) + } + + conn := proxyConnection{ + Name: cr.Name, + Local: proxyLocal{ + ConnectionType: "tcp", + TCPClient: proxyTCPEndpoint{Address: localFrontendAddress}, + TCPServer: proxyTCPEndpoint{Address: fmt.Sprintf("0.0.0.0:%d", ProxyTCPServerPort)}, + }, + Remote: remote, + ACLPolicy: buildACLPolicy(cr), + } + if t := cr.Spec.Translation; t != nil { + if len(t.Namespaces) > 0 { + nt := &proxyNamespaceTranslation{} + for _, m := range t.Namespaces { + nt.Mappings = append(nt.Mappings, proxyNamespaceMapping{Local: m.Local, Remote: m.Remote}) + } + conn.NamespaceTranslation = nt + } + if len(t.SearchAttributes) > 0 { + st := &proxySATranslation{} + for _, sa := range t.SearchAttributes { + m := proxySANamespaceMapping{Name: sa.Namespace, NamespaceID: sa.Namespace} + for _, f := range sa.Mappings { + m.Mappings = append(m.Mappings, proxyFieldMapping{LocalFieldName: f.LocalFieldName, RemoteFieldName: f.RemoteFieldName}) + } + st.NamespaceMappings = append(st.NamespaceMappings, m) + } + conn.SearchAttributeTranslation = st + } + } + if f := cr.Spec.FailoverVersionIncrement; f != nil { + conn.FailoverVersionIncrementTranslation = &proxyFailover{Local: f.Local, Remote: f.Remote} + } + + file := proxyConfigFile{ClusterConnections: []proxyConnection{conn}} + raw, err := sigsyaml.Marshal(file) + if err != nil { + return "", fmt.Errorf("marshal proxy config: %w", err) + } + return string(raw), nil +} + +func proxyRemoteCAPath(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Mux.TLS.PeerCARef != nil { + return ProxyPeerCAMountPath + "/ca.crt" + } + return ProxyTLSMountPath + "/ca.crt" +} + +func buildACLPolicy(cr *temporalv1alpha1.TemporalClusterProxy) *proxyACLPolicy { + methods := DefaultAllowedAdminMethods() + var namespaces []string + if a := cr.Spec.ACL; a != nil { + if len(a.AllowedAdminMethods) > 0 { + methods = a.AllowedAdminMethods + } + namespaces = a.AllowedNamespaces + } + return &proxyACLPolicy{ + AllowedMethods: map[string][]string{"adminService": methods}, + AllowedNamespaces: namespaces, + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `go test ./internal/resources/ -run TestBuildClusterProxyConfig -v` +Expected: PASS (both subtests). + +- [ ] **Step 5: Verify wasm build** + +Run: `GOOS=js GOARCH=wasm go build ./internal/resources/` +Expected: success (sigs.k8s.io/yaml is pure Go). + +- [ ] **Step 6: Commit** + +```bash +git add internal/resources/clusterproxyconfig.go internal/resources/clusterproxy_test.go +git commit -s -m "feat(resources): render s2s-proxy config from TemporalClusterProxy + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 3: Resource builders (pure) + +**Files:** +- Create: `internal/resources/clusterproxy.go` +- Test: `internal/resources/clusterproxy_test.go` (append) + +**Interfaces:** +- Consumes: config render + constants (Task 2). +- Produces: + - `ClusterProxyName(cr) string` → `cr.Name + "-s2s-proxy"` + - `ClusterProxyConfigMapName(cr) string`, `ClusterProxyServiceName(cr) string`, `ClusterProxyCertName(cr) string`, `ClusterProxyTLSSecretName(cr) string` + - `ClusterProxyLabels(cr) map[string]string` + - `DefaultClusterProxyImage = "temporalio/s2s-proxy:v0.2.1"` + - `func BuildClusterProxyConfigMap(cr, configYAML string) *corev1.ConfigMap` + - `func BuildClusterProxyCertificate(cr) *certmanagerv1.Certificate` (nil-safe: caller only invokes when provider=cert-manager) + - `func BuildClusterProxyDeployment(cr, configHash string) *appsv1.Deployment` + - `func BuildClusterProxyService(cr) *corev1.Service` + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/resources/clusterproxy_test.go`: + +```go +func TestBuildClusterProxyService_ServerExposesMux(t *testing.T) { + cr := serverProxyCR() + svc := resources.BuildClusterProxyService(cr) + if svc.Name != resources.ClusterProxyServiceName(cr) { + t.Errorf("service name = %q", svc.Name) + } + var haveTCP, haveMux bool + for _, p := range svc.Spec.Ports { + if p.Port == resources.ProxyTCPServerPort { + haveTCP = true + } + if p.Port == 6334 { + haveMux = true + } + } + if !haveTCP { + t.Error("expected tcpServer port 6233") + } + if !haveMux { + t.Error("expected mux port 6334 for server role") + } +} + +func TestBuildClusterProxyService_ClientOmitsMuxPort(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + cr.Spec.Mux.Server = nil + cr.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "b:6334"} + svc := resources.BuildClusterProxyService(cr) + for _, p := range svc.Spec.Ports { + if p.Name == "mux" { + t.Error("client role must not expose a mux port") + } + } +} + +func TestBuildClusterProxyDeployment_MountsConfigAndTLS(t *testing.T) { + cr := serverProxyCR() + dep := resources.BuildClusterProxyDeployment(cr, "abc123") + if dep.Name != resources.ClusterProxyName(cr) { + t.Errorf("deployment name = %q", dep.Name) + } + c := dep.Spec.Template.Spec.Containers[0] + var haveConfig, haveTLS bool + for _, m := range c.VolumeMounts { + if m.MountPath == resources.ProxyConfigMountPath { + haveConfig = true + } + if m.MountPath == resources.ProxyTLSMountPath { + haveTLS = true + } + } + if !haveConfig || !haveTLS { + t.Errorf("missing mounts: config=%v tls=%v", haveConfig, haveTLS) + } + if dep.Spec.Template.Annotations[resources.ConfigHashAnnotation] != "abc123" { + t.Errorf("config hash annotation = %q", dep.Spec.Template.Annotations[resources.ConfigHashAnnotation]) + } +} + +func TestBuildClusterProxyCertificate_UsesIssuer(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.TLS.IssuerRef = &temporalv1alpha1.IssuerReference{Name: "ca-issuer"} + crt := resources.BuildClusterProxyCertificate(cr) + if crt.Spec.IssuerRef.Name != "ca-issuer" { + t.Errorf("issuer = %q", crt.Spec.IssuerRef.Name) + } + if crt.Spec.SecretName != resources.ClusterProxyTLSSecretName(cr) { + t.Errorf("secretName = %q", crt.Spec.SecretName) + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/resources/ -run TestBuildClusterProxy -v` +Expected: FAIL — `undefined: resources.BuildClusterProxyService` (etc.). + +- [ ] **Step 3: Write the implementation** + +Create `internal/resources/clusterproxy.go` (Apache header). Note: this package's `intstrFromInt` helper already exists (used in `service.go`); reuse it. + +```go +package resources + +import ( + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +// DefaultClusterProxyImage is the pinned s2s-proxy image. +const DefaultClusterProxyImage = "temporalio/s2s-proxy:v0.2.1" + +// ClusterProxyName returns the proxy Deployment (and base) name. +func ClusterProxyName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return cr.Name + "-s2s-proxy" +} + +// ClusterProxyConfigMapName returns the rendered-config ConfigMap name. +func ClusterProxyConfigMapName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) + "-config" +} + +// ClusterProxyServiceName returns the proxy Service name. +func ClusterProxyServiceName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) +} + +// ClusterProxyCertName returns the cert-manager Certificate name. +func ClusterProxyCertName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) + "-tls" +} + +// ClusterProxyTLSSecretName returns the mux TLS Secret name (own material). +func ClusterProxyTLSSecretName(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Mux.TLS.Provider == "secret" && cr.Spec.Mux.TLS.SecretRef != nil { + return cr.Spec.Mux.TLS.SecretRef.Name + } + return ClusterProxyCertName(cr) +} + +// ClusterProxyLabels returns the standard label set for proxy resources. +func ClusterProxyLabels(cr *temporalv1alpha1.TemporalClusterProxy) map[string]string { + return map[string]string{ + LabelName: "s2s-proxy", + LabelInstance: cr.Name, + LabelComponent: "s2s-proxy", + LabelManagedBy: managedByValue, + } +} + +func clusterProxyImage(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Image != "" { + return cr.Spec.Image + } + return DefaultClusterProxyImage +} + +// BuildClusterProxyConfigMap wraps the rendered config YAML in a ConfigMap. +func BuildClusterProxyConfigMap(cr *temporalv1alpha1.TemporalClusterProxy, configYAML string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyConfigMapName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr)}, + Data: map[string]string{ProxyConfigFileName: configYAML}, + } +} + +// BuildClusterProxyCertificate builds the mux mTLS Certificate. Only call when +// mux.tls.provider is cert-manager (IssuerRef set). +func BuildClusterProxyCertificate(cr *temporalv1alpha1.TemporalClusterProxy) *certmanagerv1.Certificate { + var dnsNames []string + svc := ClusterProxyServiceName(cr) + dnsNames = append(dnsNames, + svc, + svc+"."+cr.Namespace+".svc", + svc+"."+cr.Namespace+".svc.cluster.local", + ) + return &certmanagerv1.Certificate{ + TypeMeta: metav1.TypeMeta{APIVersion: "cert-manager.io/v1", Kind: "Certificate"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyCertName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr)}, + Spec: certmanagerv1.CertificateSpec{ + SecretName: ClusterProxyTLSSecretName(cr), + CommonName: ClusterProxyName(cr), + DNSNames: dnsNames, + IssuerRef: issuerRef(cr.Spec.Mux.TLS.IssuerRef), + Usages: []certmanagerv1.KeyUsage{certmanagerv1.UsageServerAuth, certmanagerv1.UsageClientAuth}, + }, + } +} + +// BuildClusterProxyService builds the proxy Service. It always exposes the +// tcpServer port (ClusterIP, for the local Temporal) and, for the server role, +// the mux listen port using the configured exposure. +func BuildClusterProxyService(cr *temporalv1alpha1.TemporalClusterProxy) *corev1.Service { + ports := []corev1.ServicePort{ + {Name: "tcp-server", Port: ProxyTCPServerPort, TargetPort: intstrFromInt(ProxyTCPServerPort)}, + } + svcType := corev1.ServiceTypeClusterIP + var annotations map[string]string + if cr.Spec.Mux.Role == temporalv1alpha1.ProxyRoleServer && cr.Spec.Mux.Server != nil { + ports = append(ports, corev1.ServicePort{ + Name: "mux", Port: cr.Spec.Mux.Server.ListenPort, TargetPort: intstrFromInt(cr.Spec.Mux.Server.ListenPort), + }) + if ex := cr.Spec.Mux.Server.Exposure; ex != nil { + if ex.Type != "" { + svcType = ex.Type + } + annotations = ex.Annotations + } + } + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyServiceName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr), Annotations: annotations}, + Spec: corev1.ServiceSpec{ + Type: svcType, + Selector: ClusterProxyLabels(cr), + Ports: ports, + }, + } +} + +// BuildClusterProxyDeployment builds the s2s-proxy Deployment. configHash stamps +// the pod template so config/cert changes trigger a rollout. +func BuildClusterProxyDeployment(cr *temporalv1alpha1.TemporalClusterProxy, configHash string) *appsv1.Deployment { + volumes := []corev1.Volume{ + {Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: ClusterProxyConfigMapName(cr)}, + }}}, + {Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: ClusterProxyTLSSecretName(cr), + }}}, + } + mounts := []corev1.VolumeMount{ + {Name: "config", MountPath: ProxyConfigMountPath, ReadOnly: true}, + {Name: "tls", MountPath: ProxyTLSMountPath, ReadOnly: true}, + } + if ref := cr.Spec.Mux.TLS.PeerCARef; ref != nil { + volumes = append(volumes, corev1.Volume{Name: "peer-ca", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: ref.Name}}}) + mounts = append(mounts, corev1.VolumeMount{Name: "peer-ca", MountPath: ProxyPeerCAMountPath, ReadOnly: true}) + } + + replicas := int32(1) + labels := ClusterProxyLabels(cr) + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyName(cr), Namespace: cr.Namespace, Labels: labels}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{ConfigHashAnnotation: configHash}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "s2s-proxy", + Image: clusterProxyImage(cr), + Env: []corev1.EnvVar{{Name: "CONFIG_YML", Value: ProxyConfigMountPath + "/" + ProxyConfigFileName}}, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } +} +``` + +Note: the upstream container entrypoint (`scripts/start.sh`) reads the config path from the `CONFIG_YML` env var and runs `s2s-proxy start --config $CONFIG_YML` (verified against the pinned `v0.2.1` image). The builder therefore sets `CONFIG_YML` rather than overriding the container command. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `go test ./internal/resources/ -run TestBuildClusterProxy -v` +Expected: PASS. + +- [ ] **Step 5: Full package test + wasm build** + +Run: `go test ./internal/resources/... && GOOS=js GOARCH=wasm go build ./internal/resources/` +Expected: PASS and build success. + +- [ ] **Step 6: Commit** + +```bash +git add internal/resources/clusterproxy.go internal/resources/clusterproxy_test.go +git commit -s -m "feat(resources): build s2s-proxy Deployment/Service/ConfigMap/Certificate + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 4: Reconciler + +**Files:** +- Create: `internal/controller/temporalclusterproxy_controller.go` +- Test: `internal/controller/temporalclusterproxy_controller_test.go` +- Modify: `cmd/main.go` + +**Interfaces:** +- Consumes: `resolveTarget`/`ResolvedTarget`/`ErrTargetNotFound`/`namespaceDriftRequeue`/`serverSideApply` (existing `internal/controller`), `temporal.RemoteClusterClientFactory` + `RemoteClusterClient` (existing), resource builders (Task 3). +- Produces: `type TemporalClusterProxyReconciler struct{ client.Client; Scheme *runtime.Scheme; ClientFactory temporal.RemoteClusterClientFactory }` with `Reconcile` + `SetupWithManager`. + +- [ ] **Step 1: Write the controller** + +Create `internal/controller/temporalclusterproxy_controller.go` (Apache header). The test reuses the existing `fakeRemoteClient` type and inline `temporal.RemoteClusterClientFactory` defined in `internal/controller/temporalclusterconnection_controller_test.go` (same `package controller` test build) — do not redefine them. + +```go +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" + "github.com/bmorton/temporal-operator/internal/temporal" +) + +const clusterProxyFinalizer = "temporal.bmor10.com/clusterproxy" +const proxyServicesFieldOwner = client.FieldOwner("temporal-operator-clusterproxy") + +// TemporalClusterProxyReconciler deploys an s2s-proxy for one local cluster and +// registers the peer as a remote cluster via the local proxy. +type TemporalClusterProxyReconciler struct { + client.Client + Scheme *runtime.Scheme + ClientFactory temporal.RemoteClusterClientFactory +} + +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services;configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete + +func (r *TemporalClusterProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var cr temporalv1alpha1.TemporalClusterProxy + if err := r.Get(ctx, req.NamespacedName, &cr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !cr.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.reconcileDelete(ctx, &cr) + } + if !controllerutil.ContainsFinalizer(&cr, clusterProxyFinalizer) { + controllerutil.AddFinalizer(&cr, clusterProxyFinalizer) + if err := r.Update(ctx, &cr); err != nil { + return ctrl.Result{}, err + } + } + + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err != nil { + if errors.Is(err, ErrTargetNotFound) { + r.setReady(&cr, metav1.ConditionFalse, temporalv1alpha1.ReasonClusterNotReady, "local cluster not found") + return ctrl.Result{RequeueAfter: namespaceDriftRequeue}, r.statusUpdate(ctx, &cr) + } + return ctrl.Result{}, err + } + + // Render + apply proxy resources. + configYAML, err := resources.BuildClusterProxyConfig(&cr, target.Address) + if err != nil { + return ctrl.Result{}, err + } + sum := sha256.Sum256([]byte(configYAML)) + configHash := hex.EncodeToString(sum[:]) + + objs := []client.Object{ + resources.BuildClusterProxyConfigMap(&cr, configYAML), + resources.BuildClusterProxyService(&cr), + resources.BuildClusterProxyDeployment(&cr, configHash), + } + if cr.Spec.Mux.TLS.Provider != "secret" && cr.Spec.Mux.TLS.IssuerRef != nil { + objs = append([]client.Object{resources.BuildClusterProxyCertificate(&cr)}, objs...) + } + for _, obj := range objs { + if err := r.apply(ctx, &cr, obj); err != nil { + return ctrl.Result{}, err + } + } + + deployReady := r.deploymentAvailable(ctx, &cr) + if deployReady { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionProxyDeployed, Status: metav1.ConditionTrue, Reason: temporalv1alpha1.ReasonProxyReady, Message: "proxy deployment available", ObservedGeneration: cr.Generation}) + } else { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionProxyDeployed, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonProxyNotReady, Message: "proxy deployment not yet available", ObservedGeneration: cr.Generation}) + } + + // Publish endpoint for server role. + if cr.Spec.Mux.Role == temporalv1alpha1.ProxyRoleServer && cr.Spec.Mux.Server != nil { + cr.Status.ProxyEndpoint = r.serverEndpoint(ctx, &cr) + } + + // Register the peer via the local proxy once the proxy and local cluster are ready. + registered := false + if deployReady && target.Ready { + if err := r.registerPeer(ctx, &cr, target.TLSConfig); err != nil { + logf.FromContext(ctx).Error(err, "registering peer via proxy") + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonRegistrationFailed, Message: err.Error(), ObservedGeneration: cr.Generation}) + } else { + registered = true + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionTrue, Reason: temporalv1alpha1.ReasonProxyReady, Message: "peer registered via local proxy", ObservedGeneration: cr.Generation}) + } + } else { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonClusterNotReady, Message: "waiting for proxy and local cluster", ObservedGeneration: cr.Generation}) + } + + if deployReady && registered { + r.setReady(&cr, metav1.ConditionTrue, temporalv1alpha1.ReasonProxyReady, "proxy deployed and peer registered") + } else { + r.setReady(&cr, metav1.ConditionFalse, temporalv1alpha1.ReasonProxyNotReady, "proxy not fully converged") + } + return ctrl.Result{RequeueAfter: namespaceDriftRequeue}, r.statusUpdate(ctx, &cr) +} + +// registerPeer dials the local Temporal frontend and registers the peer with the +// local proxy tcpServer address as its frontend. +func (r *TemporalClusterProxyReconciler) registerPeer(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy, tlsConfig interface{ /* *tls.Config */ }) error { + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err != nil { + return err + } + c, err := r.clientFactory()(ctx, target.Address, target.TLSConfig) + if err != nil { + return err + } + defer func() { _ = c.Close() }() + proxyAddr := fmt.Sprintf("%s.%s.svc.cluster.local:%d", resources.ClusterProxyServiceName(cr), cr.Namespace, resources.ProxyTCPServerPort) + enable := cr.Spec.Peer.EnableConnection == nil || *cr.Spec.Peer.EnableConnection + return c.UpsertRemoteCluster(ctx, proxyAddr, enable) +} + +func (r *TemporalClusterProxyReconciler) reconcileDelete(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + if !controllerutil.ContainsFinalizer(cr, clusterProxyFinalizer) { + return nil + } + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err == nil && target.Ready { + if c, derr := r.clientFactory()(ctx, target.Address, target.TLSConfig); derr == nil { + _ = c.RemoveRemoteCluster(ctx, cr.Spec.Peer.Name) + _ = c.Close() + } + } + // Owned resources are GC'd via owner references. + controllerutil.RemoveFinalizer(cr, clusterProxyFinalizer) + return r.Update(ctx, cr) +} + +func (r *TemporalClusterProxyReconciler) deploymentAvailable(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) bool { + var dep appsv1.Deployment + key := types.NamespacedName{Namespace: cr.Namespace, Name: resources.ClusterProxyName(cr)} + if err := r.Get(ctx, key, &dep); err != nil { + return false + } + return dep.Status.AvailableReplicas > 0 +} + +func (r *TemporalClusterProxyReconciler) serverEndpoint(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) string { + // For LoadBalancer, surface the assigned ingress; otherwise the in-cluster DNS name. + var svc corev1Service + _ = svc // resolved below + return fmt.Sprintf("%s.%s.svc.cluster.local:%d", resources.ClusterProxyServiceName(cr), cr.Namespace, cr.Spec.Mux.Server.ListenPort) +} + +func (r *TemporalClusterProxyReconciler) apply(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy, obj client.Object) error { + if err := controllerutil.SetControllerReference(cr, obj, r.Scheme); err != nil { + return err + } + return serverSideApply(ctx, r.Client, r.Scheme, obj, proxyServicesFieldOwner) +} + +func (r *TemporalClusterProxyReconciler) clientFactory() temporal.RemoteClusterClientFactory { + if r.ClientFactory != nil { + return r.ClientFactory + } + return temporal.NewRemoteClusterClient +} + +func (r *TemporalClusterProxyReconciler) setReady(cr *temporalv1alpha1.TemporalClusterProxy, status metav1.ConditionStatus, reason, message string) { + cr.Status.ObservedGeneration = cr.Generation + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: temporalv1alpha1.ConditionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: cr.Generation, + }) +} + +func (r *TemporalClusterProxyReconciler) statusUpdate(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + return client.IgnoreNotFound(r.Status().Update(ctx, cr)) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TemporalClusterProxyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&temporalv1alpha1.TemporalClusterProxy{}). + Owns(&appsv1.Deployment{}). + Named("temporalclusterproxy"). + Complete(r) +} +``` + +**IMPLEMENTATION FIXUPS (do these while writing, they are deliberate simplifications above):** +1. Remove the unused `tlsConfig interface{...}` parameter from `registerPeer` — it re-resolves the target itself, so the signature is `registerPeer(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error`, and the caller is `r.registerPeer(ctx, &cr)`. +2. Replace the placeholder `corev1Service` in `serverEndpoint` with a real read: `import corev1 "k8s.io/api/core/v1"`, `var svc corev1.Service; if err := r.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: resources.ClusterProxyServiceName(cr)}, &svc); err == nil { for _, ing := range svc.Status.LoadBalancer.Ingress { host := ing.Hostname; if host == "" { host = ing.IP }; if host != "" { return fmt.Sprintf("%s:%d", host, cr.Spec.Mux.Server.ListenPort) } } }` then fall through to the DNS name. +3. Drop the unused `logf` import if only used once — keep it; it is used in `registerPeer`'s caller. + +- [ ] **Step 2: Write the failing envtest** + +Create `internal/controller/temporalclusterproxy_controller_test.go` following the structure of `temporalclusterconnection_controller_test.go` (Ginkgo, uses the shared `suite_test.go` `k8sClient`/`cfg`). Minimum coverage: + +```go +package controller + +// Ginkgo spec (mirror temporalclusterconnection_controller_test.go layout): +// It("deploys the proxy and sets conditions", func() { +// - create a TemporalCluster "cluster-a" (or a minimal Ready one, as the +// connection test does) in a fresh namespace +// - create a TemporalClusterProxy referencing it, role=server, listenPort 6334, +// tls.provider=secret + secretRef to a pre-created Secret (avoids needing +// cert-manager in envtest) +// - reconcile via a TemporalClusterProxyReconciler{Client, Scheme, ClientFactory: fakeFactory} +// - Eventually: a Deployment named cluster-a-s2s-proxy exists, a ConfigMap and +// Service exist, all with an ownerRef to the proxy CR +// - the ProxyDeployed condition is present +// }) +``` + +Use the existing `fakeRemoteClient` and factory pattern from `temporalclusterconnection_controller_test.go` (lines ~49–90). If the connection test constructs `k8sClient` objects via helper builders, reuse them. Provide a BYO TLS Secret so no cert-manager dependency is needed: + +```go +tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "mux-tls", Namespace: ns}, + Data: map[string][]byte{"tls.crt": []byte("x"), "tls.key": []byte("y"), "ca.crt": []byte("z")}, +} +``` + +and set `cr.Spec.Mux.TLS = temporalv1alpha1.ProxyMuxTLS{Provider: "secret", SecretRef: &temporalv1alpha1.SecretReference{Name: "mux-tls"}}`. + +- [ ] **Step 3: Run to verify failure** + +Run: `make test` (envtest). Expected: the new spec FAILS (reconciler/objects not created) while the rest pass. If compilation fails first, fix compile errors, then observe the intended assertion failure. + +- [ ] **Step 4: Make the test pass** + +Apply the fixups from Step 1; ensure the reconciler compiles and the spec passes. + +Run: `make test` +Expected: PASS (whole suite). + +- [ ] **Step 5: Register in main.go** + +In `cmd/main.go`, after the `TemporalClusterConnectionReconciler` block (around line 251), add: + +```go + if err := (&controller.TemporalClusterProxyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TemporalClusterProxy") + os.Exit(1) + } +``` + +- [ ] **Step 6: Regenerate RBAC + chart** + +Run: `make manifests helm-chart` +Expected: `config/rbac/role.yaml` gains the new rules; `dist/chart` updated. Remove `.github/workflows/test-chart.yml` if it reappears. + +- [ ] **Step 7: Build + commit** + +Run: `go build ./... && make lint` +Expected: success, no lint errors. + +```bash +git add internal/controller/ cmd/main.go config/ dist/chart/ +git commit -s -m "feat(controller): reconcile TemporalClusterProxy (deploy + register) + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 5: Webhook validation + defaulting + +**Files:** +- Create: `internal/webhook/v1alpha1/temporalclusterproxy_webhook.go` +- Test: `internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go` +- Modify: `cmd/main.go` + +**Interfaces:** +- Consumes: `TemporalClusterProxy` types. +- Produces: `SetupTemporalClusterProxyWebhookWithManager(mgr) error`; `TemporalClusterProxyCustomValidator`; `TemporalClusterProxyCustomDefaulter`. + +- [ ] **Step 1: Write the failing test** + +Create `internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go` (mirror `temporalclusterconnection_webhook_test.go`). Cases: + +```go +// server role without server block -> error +// client role without client block -> error +// tls provider cert-manager without issuerRef -> error +// tls provider secret without secretRef -> error +// peer.name == localClusterName -> error +// valid server CR -> no error +// defaulter fills tls.provider=cert-manager, peer.enableConnection=true, image +``` + +Write these as plain Go table tests calling `(&TemporalClusterProxyCustomValidator{}).ValidateCreate(ctx, cr)` and `(&TemporalClusterProxyCustomDefaulter{}).Default(ctx, cr)`. + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/webhook/v1alpha1/ -run TestTemporalClusterProxy -v` +Expected: FAIL — undefined validator/defaulter. + +- [ ] **Step 3: Write the webhook** + +Create `internal/webhook/v1alpha1/temporalclusterproxy_webhook.go` (Apache header): + +```go +package v1alpha1 + +import ( + "context" + + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" +) + +var temporalclusterproxylog = logf.Log.WithName("temporalclusterproxy-resource") + +// SetupTemporalClusterProxyWebhookWithManager registers the webhook. +func SetupTemporalClusterProxyWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr, &temporalv1alpha1.TemporalClusterProxy{}). + WithValidator(&TemporalClusterProxyCustomValidator{}). + WithDefaulter(&TemporalClusterProxyCustomDefaulter{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-temporal-bmor10-com-v1alpha1-temporalclusterproxy,mutating=false,failurePolicy=fail,sideEffects=None,groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=create;update,versions=v1alpha1,name=vtemporalclusterproxy-v1alpha1.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-temporal-bmor10-com-v1alpha1-temporalclusterproxy,mutating=true,failurePolicy=fail,sideEffects=None,groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=create;update,versions=v1alpha1,name=mtemporalclusterproxy-v1alpha1.kb.io,admissionReviewVersions=v1 + +// TemporalClusterProxyCustomDefaulter defaults optional fields. +type TemporalClusterProxyCustomDefaulter struct{} + +var _ admission.Defaulter[*temporalv1alpha1.TemporalClusterProxy] = &TemporalClusterProxyCustomDefaulter{} + +func (d *TemporalClusterProxyCustomDefaulter) Default(_ context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + if cr.Spec.Mux.TLS.Provider == "" { + cr.Spec.Mux.TLS.Provider = "cert-manager" + } + if cr.Spec.Peer.EnableConnection == nil { + enable := true + cr.Spec.Peer.EnableConnection = &enable + } + if cr.Spec.Image == "" { + cr.Spec.Image = resources.DefaultClusterProxyImage + } + return nil +} + +// TemporalClusterProxyCustomValidator validates the CR. +type TemporalClusterProxyCustomValidator struct{} + +var _ admission.Validator[*temporalv1alpha1.TemporalClusterProxy] = &TemporalClusterProxyCustomValidator{} + +func validateClusterProxy(cr *temporalv1alpha1.TemporalClusterProxy) field.ErrorList { + var errs field.ErrorList + specPath := field.NewPath("spec") + + if cr.Spec.LocalClusterRef.Name == "" { + errs = append(errs, field.Required(specPath.Child("localClusterRef", "name"), "local cluster reference is required")) + } + if cr.Spec.Peer.Name == "" { + errs = append(errs, field.Required(specPath.Child("peer", "name"), "peer name is required")) + } + if cr.Spec.Peer.Name != "" && cr.Spec.Peer.Name == cr.Spec.LocalClusterName { + errs = append(errs, field.Invalid(specPath.Child("peer", "name"), cr.Spec.Peer.Name, "peer name must differ from localClusterName")) + } + + muxPath := specPath.Child("mux") + switch cr.Spec.Mux.Role { + case temporalv1alpha1.ProxyRoleServer: + if cr.Spec.Mux.Server == nil { + errs = append(errs, field.Required(muxPath.Child("server"), "server is required for role=server")) + } + if cr.Spec.Mux.Client != nil { + errs = append(errs, field.Invalid(muxPath.Child("client"), "client", "client must be unset for role=server")) + } + case temporalv1alpha1.ProxyRoleClient: + if cr.Spec.Mux.Client == nil || cr.Spec.Mux.Client.ServerAddress == "" { + errs = append(errs, field.Required(muxPath.Child("client", "serverAddress"), "client.serverAddress is required for role=client")) + } + if cr.Spec.Mux.Server != nil { + errs = append(errs, field.Invalid(muxPath.Child("server"), "server", "server must be unset for role=client")) + } + default: + errs = append(errs, field.NotSupported(muxPath.Child("role"), cr.Spec.Mux.Role, []string{temporalv1alpha1.ProxyRoleServer, temporalv1alpha1.ProxyRoleClient})) + } + + tlsPath := muxPath.Child("tls") + switch cr.Spec.Mux.TLS.Provider { + case "", "cert-manager": + if cr.Spec.Mux.TLS.IssuerRef == nil || cr.Spec.Mux.TLS.IssuerRef.Name == "" { + errs = append(errs, field.Required(tlsPath.Child("issuerRef"), "issuerRef is required for provider=cert-manager")) + } + case "secret": + if cr.Spec.Mux.TLS.SecretRef == nil || cr.Spec.Mux.TLS.SecretRef.Name == "" { + errs = append(errs, field.Required(tlsPath.Child("secretRef"), "secretRef is required for provider=secret")) + } + default: + errs = append(errs, field.NotSupported(tlsPath.Child("provider"), cr.Spec.Mux.TLS.Provider, []string{"cert-manager", "secret"})) + } + return errs +} + +func (v *TemporalClusterProxyCustomValidator) ValidateCreate(_ context.Context, cr *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + temporalclusterproxylog.Info("validate create", "name", cr.GetName()) + if errs := validateClusterProxy(cr); len(errs) > 0 { + return nil, errs.ToAggregate() + } + return nil, nil +} + +func (v *TemporalClusterProxyCustomValidator) ValidateUpdate(_ context.Context, oldCR, newCR *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + temporalclusterproxylog.Info("validate update", "name", newCR.GetName()) + var errs field.ErrorList + if oldCR.Spec.Mux.Role != newCR.Spec.Mux.Role { + errs = append(errs, field.Forbidden(field.NewPath("spec", "mux", "role"), "mux.role is immutable")) + } + if oldCR.Spec.LocalClusterRef.Name != newCR.Spec.LocalClusterRef.Name { + errs = append(errs, field.Forbidden(field.NewPath("spec", "localClusterRef"), "localClusterRef is immutable")) + } + errs = append(errs, validateClusterProxy(newCR)...) + if len(errs) > 0 { + return nil, errs.ToAggregate() + } + return nil, nil +} + +func (v *TemporalClusterProxyCustomValidator) ValidateDelete(_ context.Context, _ *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + return nil, nil +} +``` + +Note: if the `Defaulter` mutating webhook causes an import cycle or the webhook package must not import `internal/resources`, inline the default image string constant instead of importing `resources`. Check whether other webhooks import `internal/resources`; if not, define a local `const defaultProxyImage = "temporalio/s2s-proxy:v0.2.1"` and use it in the defaulter. + +- [ ] **Step 4: Run to verify pass** + +Run: `go test ./internal/webhook/v1alpha1/ -run TestTemporalClusterProxy -v` +Expected: PASS. + +- [ ] **Step 5: Register webhook in main.go** + +In `cmd/main.go`, add another `webhooksEnabled` block after the connection webhook (around line 282): + +```go + if webhooksEnabled { + if err := webhookv1alpha1.SetupTemporalClusterProxyWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "TemporalClusterProxy") + os.Exit(1) + } + } +``` + +- [ ] **Step 6: Regenerate + full validation** + +Run: `make manifests generate helm-chart && make lint test build` +Expected: all green. `config/webhook/manifests.yaml` gains the new webhook entries; remove `.github/workflows/test-chart.yml` if it reappears. + +- [ ] **Step 7: Commit** + +```bash +git add internal/webhook/ cmd/main.go config/ dist/chart/ +git commit -s -m "feat(webhook): validate and default TemporalClusterProxy + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 6: Examples + docs + e2e + +**Files:** +- Create: `examples/cluster-proxy-mux/server.yaml`, `examples/cluster-proxy-mux/client.yaml` +- Create: `test/e2e/clusterproxy/chainsaw-test.yaml` (+ supporting manifests) +- Modify: docs/examples index if one exists (follow the pattern in `docs/content/examples` — check first). + +- [ ] **Step 1: Write the example CRs** + +`examples/cluster-proxy-mux/server.yaml`: + +```yaml +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: to-cluster-a + namespace: temporal-system +spec: + localClusterRef: + name: cluster-b + localClusterName: cluster-b + peer: + name: cluster-a + mux: + role: server + server: + listenPort: 6334 + exposure: + type: LoadBalancer + tls: + provider: cert-manager + issuerRef: + name: temporal-ca +``` + +`examples/cluster-proxy-mux/client.yaml`: + +```yaml +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: to-cluster-b + namespace: temporal-system +spec: + localClusterRef: + name: cluster-a + localClusterName: cluster-a + peer: + name: cluster-b + mux: + role: client + client: + serverAddress: cluster-b-proxy.example.com:6334 + tls: + provider: cert-manager + issuerRef: + name: temporal-ca + peerCARef: + name: cluster-b-mux-ca +``` + +- [ ] **Step 2: Validate examples against the CRD (dry run)** + +Run (against a cluster or with `kubectl --dry-run=client` after `kubectl apply -f config/crd/bases`): confirm both manifests pass schema validation. If no cluster is available, run `go test ./internal/webhook/v1alpha1/ -run TestTemporalClusterProxy` after loading these YAMLs via a small test, or validate manually with the webhook validator function. + +- [ ] **Step 3: Write the chainsaw e2e** + +Create `test/e2e/clusterproxy/chainsaw-test.yaml` following the layout of `test/e2e/devserver/chainsaw-test.yaml` and `test/e2e/migration/` (both are multi-step replication tests). The e2e should: stand up two clusters (or one cluster + one dev server if single-network is acceptable for CI), apply a server + client `TemporalClusterProxy`, wait for both to reach `Ready=True`, register a global namespace, and assert replication of a workflow across the link. Reuse the existing migration e2e's worker/namespace manifests as a template. + +Because two-network mux is hard to reproduce in a single kind cluster, the CI variant may run both proxies in one cluster (server role reachable via its ClusterIP Service, client `serverAddress` = that Service DNS) to exercise the full path deterministically. + +- [ ] **Step 4: Run the e2e locally** + +Run the project's e2e entrypoint (check `Makefile` for the chainsaw target, e.g. `make test-e2e` or the nsc runner). Expected: the clusterproxy suite passes. If the two-cluster topology is infeasible in the available CI, mark the suite as the deterministic single-cluster variant described in Step 3 and note the true cross-network path is validated manually. + +- [ ] **Step 5: Final full validation** + +Run: `make generate manifests api-docs docs-crd-reference helm-chart && make lint test build` +Then `git status`: ensure only intended files changed; remove `.github/workflows/test-chart.yml` if present/untracked. + +- [ ] **Step 6: Commit** + +```bash +git add examples/ test/e2e/clusterproxy/ docs/ +git commit -s -m "test(e2e): TemporalClusterProxy mux replication + examples + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Self-review notes (already applied) + +- **Spec coverage:** API (Task 1) ↔ spec §API; rendering (Task 2) ↔ §Config rendering; builders (Task 3) ↔ §Rendered resources; reconcile/registration/finalizer/status (Task 4) ↔ §Reconciliation + §Error handling; validation/defaulting (Task 5) ↔ §Webhook validation; testing + examples + e2e (Tasks 2–6) ↔ §Testing + §Delivery; chart/docs regen folded into each task per §Delivery. +- **Known simplifications flagged inline for the implementer:** the `registerPeer` signature and the `serverEndpoint` LoadBalancer read in Task 4 Step 1 are marked as fixups to apply while writing (the first-draft code intentionally shows the shape, the fixups make it compile). +- **Type consistency:** builder/name helpers (`ClusterProxyName`, `ClusterProxyServiceName`, `ClusterProxyTLSSecretName`, `ProxyTCPServerPort`, `ProxyTLSMountPath`) are defined in Tasks 2–3 and referenced verbatim in Task 4. Condition/reason constants are defined in Task 1 Step 2 and used in Task 4. +- **External assumptions already verified against the pinned `v0.2.1` release:** image tag (`temporalio/s2s-proxy:v0.2.1`), the container entrypoint (`CONFIG_YML` env var → `s2s-proxy start --config`), and the config keys used by the render func (`clusterConnections`, `tcpClient`/`tcpServer`, `mux-server`/`mux-client`, `muxAddressInfo.tls`, `namespaceTranslation`, `searchAttributeTranslation`, `failoverVersionIncrementTranslation`, `aclPolicy`) — all present in the upstream `develop/config` examples at that ref. diff --git a/docs/superpowers/specs/2026-07-03-temporalclusterproxy-s2s-proxy-design.md b/docs/superpowers/specs/2026-07-03-temporalclusterproxy-s2s-proxy-design.md new file mode 100644 index 0000000..375e3de --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-temporalclusterproxy-s2s-proxy-design.md @@ -0,0 +1,252 @@ +# TemporalClusterProxy: automating s2s-proxy for cross-network replication + +Status: Draft +Date: 2026-07-03 +Related: `2026-06-23-multi-cluster-replication-automation-design.md` (the direct-network +`TemporalClusterConnection` CRD this builds alongside). + +## Summary + +Add a new `TemporalClusterProxy` CRD that automates deployment and wiring of +[temporalio/s2s-proxy](https://github.com/temporalio/s2s-proxy) so operator-managed +Temporal clusters can replicate across **segregated networks** without exposing their +frontends to each other. The headline use case is cross-network disaster recovery over +s2s-proxy **mux mode** with mTLS; the CRD is designed as a general foundation that also +carries s2s-proxy's namespace / search-attribute translation, failover-version +translation, and ACL allowlists. + +## Motivation + +Today `TemporalClusterConnection` automates multi-cluster replication by having a single +operator dial **both** peers' Temporal frontends directly and call +`AddOrUpdateRemoteCluster`. That assumes the peers share a routable network and each +frontend is directly reachable. It cannot serve clusters that must not share a network +(different VPCs / accounts / on-prem ↔ cloud), where only one side can open a port. + +s2s-proxy solves exactly this: in **mux mode** one side opens a port (mux-server) and the +other dials out (mux-client), multiplexing bidirectional replication over a single +TCP-TLS connection, so the dialing side never exposes a port. It also adds in-flight +namespace / search-attribute translation, per-API / per-namespace ACLs, failover-version +translation, and cross-version protobuf adaptation. Automating it removes a large amount +of fiddly, error-prone manual config (proxy YAML, cert wiring, remote-cluster +registration) and brings it under the operator's reconcile/status model. + +## Non-goals + +- Replacing `TemporalClusterConnection`. It remains the direct-network option; + `TemporalClusterProxy` is the segregated-network option. +- TCP (non-mux) proxy mode and pure same-network namespace migration. The API leaves room + for it, but the first deliverable targets mux DR. +- Auto-discovering the peer's exposed mux address across clusters. In the both-operator + case the user copies the server's `status.proxyEndpoint` into the client CR. + +## Deployment / topology model + +The CR describes **one local cluster + its proxy + one peer link**. Both replication +directions ride the single mux, so there is one `TemporalClusterProxy` per cluster per +peer. For a symmetric, both-operator-managed DR pair the user authors two CRs — one +`mux.role: server` (in the cluster that exposes a port) and one `mux.role: client` (in the +cluster that dials out), one in each cluster's operator. The two operators do not talk to +each other; they converge independently from the matching CRs. + +The peer may be: +- **operator-managed** (`peer.clusterRef` set) — used to reuse the peer's issuer CA + automatically, or +- **external** (no `clusterRef`) — the remote proxy/cluster is run by someone else; its CA + is supplied via `mux.tls.peerCARef`. + +### Data flow (A = mux-client, B = mux-server) + +1. Operator registers remote cluster **B** on **A's** Temporal via + `AddOrUpdateRemoteCluster`, with `frontendAddress` = **A's own proxy** `tcpServer` + Service (cluster-internal). This reuses the existing `internal/temporal` + `RemoteClusterClient`; the only change from `TemporalClusterConnection` is that the + address points at the local proxy instead of the remote frontend. +2. A's Temporal streams replication → A-proxy `tcpServer` → mux → B-proxy → B's Temporal + frontend (`tcpClient`). +3. The reverse direction (B → A) rides the same mux; B's CR registers **A** on B's + Temporal via B's local proxy. + +## API design + +New namespaced CRD, group `temporal.bmor10.com/v1alpha1`, shortName `tcproxy`, marked +`storageversion`, with a `status` subresource. + +### Spec + +```yaml +spec: + localClusterRef: # required — the local operator-managed TemporalCluster this proxy fronts + name: cluster-a + kind: TemporalCluster # reuses existing ClusterReference (defaults to TemporalCluster) + # localClusterName: optional override; else derived from the cluster's + # clusterMetadata.currentClusterName + + peer: # the remote replication cluster on the far side of the mux + name: cluster-b # == that cluster's currentClusterName; must != local cluster name + clusterRef: # optional: set when the remote is also operator-managed (CA reuse only) + name: cluster-b + kind: TemporalCluster + enableConnection: true # default true; toggle replication without deleting the CR + + mux: + role: server | client # required — one side opens a port, the other dials out + server: # required when role=server + listenPort: 6334 + exposure: {...} # reuses existing ServiceExposureSpec (LoadBalancer/NodePort/...) + client: # required when role=client + serverAddress: b.example.com:6334 + muxCount: 4 # optional; defaults to upstream default + tls: + provider: cert-manager | secret # default cert-manager (mirrors MTLSSpec) + issuerRef: {...} # cert-manager path — mints this side's mux cert + secretRef: {...} # BYO path (cert/key/CA); 1Password OnePasswordItem-friendly + peerCARef: {...} # remote CA to trust; auto-reused from peer.clusterRef issuer when available + + # --- general-foundation extras, all optional --- + translation: + namespaces: + - { local: ns, remote: ns.accountid } + searchAttributes: + - namespace: ns + mappings: + - { localFieldName: x, remoteFieldName: y } + failoverVersionIncrement: { local: 100, remote: 1000 } + acl: + allowedNamespaces: [ns] + allowedAdminMethods: [...] # defaults to the known replication allowlist + + image: temporalio/s2s-proxy: # pinned default; overridable (native render, per decision) +``` + +### Status + +```yaml +status: + observedGeneration: N + proxyEndpoint: "1.2.3.4:6334" # server role: the address to hand to the peer's client CR + conditions: + - type: Ready + - type: ProxyDeployed # proxy Deployment available + - type: MTLSReady # cert issued / BYO secret present & valid + - type: RemoteClusterRegistered # local Temporal registered the peer via the local proxy +``` + +## Rendered resources + +All owned by the CR via `ownerRef` (GC'd on delete), built by **pure** +`internal/resources` `Build*` functions (no client/IO, so they stay `GOOS=js +GOARCH=wasm`-compatible, matching the existing convention). + +| Resource | Purpose | +|---|---| +| `ConfigMap` | Rendered s2s-proxy `clusterConnections` YAML (pure render func, mirroring `internal/temporal/configtemplate.go`) | +| `Certificate` (cert-manager) | Only when `tls.provider: cert-manager`; mints this side's mux cert → Secret | +| `Deployment` | `temporalio/s2s-proxy` pod; mounts config ConfigMap + TLS Secret; `--config` arg | +| `Service` | Two ports: **tcpServer** (ClusterIP, for local Temporal to reach) + **mux** (server role only, exposed per `mux.server.exposure`) | + +### Config rendering + +Maps the spec to one `clusterConnections` entry: + +- `local.connectionType: tcp`; `local.tcpClient.address` = resolved local Temporal frontend + Service; `local.tcpServer.address` = `0.0.0.0:`. +- `remote.connectionType: mux-server | mux-client`; `muxAddressInfo` = listen address + (server) or `serverAddress` (client), with TLS `certificatePath` / `keyPath` / + `remoteCAPath` pointing at the mounted Secret. +- Optional `namespaceTranslation`, `searchAttributeTranslation`, + `failoverVersionIncrementTranslation`, and `aclPolicy` rendered from the spec. `aclPolicy` + defaults to the known replication admin-method allowlist + (`AddOrUpdateRemoteCluster`, `DescribeCluster`, `DescribeMutableState`, + `GetNamespaceReplicationMessages`, `GetWorkflowExecutionRawHistoryV2`, `ListClusters`, + `StreamWorkflowReplicationMessages`). + +## Reconciliation + +One CR = one local cluster's proxy. The loop: + +1. Resolve `localClusterRef` → frontend address + issuer CA (reuse `resolveTarget`). +2. Ensure TLS: create the `Certificate` (cert-manager) or validate the BYO Secret keys → + set `MTLSReady`. +3. Render + apply ConfigMap, Deployment, Service via the existing server-side-apply + `r.apply` pattern → set `ProxyDeployed` when the Deployment is available. +4. When the proxy is ready **and** the local Temporal is ready: `AddOrUpdateRemoteCluster` + pointing at the local proxy `tcpServer` Service → set `RemoteClusterRegistered`. +5. Publish `status.proxyEndpoint` (server role), set `Ready`, requeue on drift + (`namespaceDriftRequeue`). +6. **Finalizer:** on delete, best-effort `RemoveRemoteCluster` from the local Temporal, + let ownerRef GC the proxy resources, then remove the finalizer — mirroring + `TemporalClusterConnection` so deletion always converges even if Temporal is + unreachable. + +The `RemoteClusterClientFactory` is injectable for tests, as in the existing connection +controller. + +## Webhook validation + +Typed `admission.Validator[*TemporalClusterProxy]` (+ `Defaulter`), per the +controller-runtime v0.23 generic admission convention already used in this repo. + +- `mux.role: server` requires `mux.server.listenPort` (and exposure); `role: client` + requires `mux.client.serverAddress`. Reject wrong-role fields being set. +- `tls.provider: cert-manager` requires `issuerRef`; `provider: secret` requires + `secretRef`. +- `peer.name` required and `!=` local cluster name; `localClusterRef` required. +- Immutable after creation: `mux.role`, `localClusterRef`. +- Defaulter fills: `tls.provider` (cert-manager), `peer.enableConnection` (true), `image`. + +## Error handling + +Per-condition, eventual-consistency; a single failing dependency never wedges the whole +reconcile. + +- Local cluster missing / not ready → `RemoteClusterRegistered=False`, `Ready=False` + (`ReasonClusterNotReady`), requeue; registration untouched. +- cert-manager Certificate not yet issued / BYO Secret missing keys → `MTLSReady=False`. +- Deployment not yet available → `ProxyDeployed=False`. +- Registration RPC fails (proxy up, Temporal transiently unreachable) → + `RemoteClusterRegistered=False`, requeue. +- Deletion always converges (best-effort de-register → GC → remove finalizer). + +## Testing + +- **Unit (pure):** config-render func across server / client / translation / ACL / + failover permutations (table tests, like `configtemplate_test.go`); the `Build*` + resource builders; keep the `wasm` build green. +- **Webhook tests:** all validation rules (envtest suite). +- **Controller envtest:** CR → owned ConfigMap / Deployment / Service / Certificate created + with ownerRefs; conditions progress; registration invoked via an injected fake + `RemoteClusterClientFactory`; finalizer de-registers on delete. +- **e2e (chainsaw, final milestone):** two kind clusters, one server + one client CR, + mTLS mux, replicate a namespace end-to-end. + +## Delivery / repo conventions + +- `make generate manifests` (deepcopy + CRD). RBAC markers for the new controller + + `deployments` / `services` / `configmaps` / `secrets` and cert-manager `certificates`. +- `make api-docs docs-crd-reference`; commit `docs/api/v1alpha1.md` + + `docs/content/reference/_index.md` (docs CI drift check). +- `make helm-chart`; commit `dist/chart` (verify-chart CI). Mirror any manager RBAC/args + into `hack/helm/overrides/` if the manager Deployment changes. +- `make lint test build` green. Conventional-commit `feat(...)`, DCO sign-off. +- Add `examples/cluster-proxy-mux/` with a server + client CR pair. + +## Milestones + +Each is independently plan-able / reviewable: + +1. **API** — types + deepcopy + CRD manifests + docs + chart regen. +2. **Rendering** — config-render func + `Build*` resource builders + unit tests. +3. **Controller** — reconcile (TLS → deploy → register) + finalizer + envtest. +4. **Webhook** — validation + defaulting + tests. +5. **e2e** — chainsaw two-cluster mux replication + example manifests. + +## Open questions / risks + +- s2s-proxy is "binary only; internal APIs subject to change." Mitigation: pin the image + and treat it like the other pinned tool/image versions in the version matrix. +- Confirm the exact upstream config keys (`muxCount`, TLS `caServerName` / + `skipCAVerification`, health-check blocks) against the pinned s2s-proxy release when + implementing the render func. +- Cross-cluster CA distribution in the both-operator external case is manual (peer CA + secret) — acceptable for the first cut. diff --git a/examples/cluster-proxy-mux/README.md b/examples/cluster-proxy-mux/README.md new file mode 100644 index 0000000..4e7b52a --- /dev/null +++ b/examples/cluster-proxy-mux/README.md @@ -0,0 +1,58 @@ +# TemporalClusterProxy mux — server + client example + +Deploys an s2s-proxy mux link between two Temporal replication clusters using +the `TemporalClusterProxy` CRD. + +## Architecture + +```text +cluster-a cluster-b +┌─────────────────────┐ ┌─────────────────────┐ +│ Temporal frontend │ │ Temporal frontend │ +│ :7233 │ │ :7233 │ +│ ↕ │ │ ↕ │ +│ client proxy │◄──mux──│ server proxy │ +│ (to-cluster-b) │ 6334 │ (to-cluster-a) │ +│ port 6233 (local) │ │ port 6233 (local) │ +└─────────────────────┘ └─────────────────────┘ +``` + +- **server.yaml** — `TemporalClusterProxy` with `role: server` deployed next to + `cluster-b`. Opens the mux port (6334) and exposes it via a LoadBalancer + Service so cluster-a's client proxy can dial in. +- **client.yaml** — `TemporalClusterProxy` with `role: client` deployed next to + `cluster-a`. Dials out to the server proxy's LoadBalancer address. + +## Prerequisites + +- Both `cluster-a` and `cluster-b` `TemporalCluster` resources are Ready. +- cert-manager is installed and a `temporal-ca` Issuer exists in `temporal-system`. +- The server proxy's LoadBalancer IP/hostname is known before applying the client. + +## Apply + +```sh +# On the cluster hosting cluster-b: apply the server proxy +kubectl apply -n temporal-system -f server.yaml + +# Retrieve the assigned LoadBalancer address: +kubectl get svc -n temporal-system to-cluster-a-s2s-proxy \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' + +# Edit client.yaml: replace cluster-b-proxy.example.com with the address above. +# On the cluster hosting cluster-a: apply the client proxy +kubectl apply -n temporal-system -f client.yaml + +# Verify both proxies are Ready: +kubectl get tcproxy -n temporal-system +``` + +## Status + +Once both proxies reach `Ready=True`, each cluster's frontend has the other +registered as a remote peer and replication can flow through the mux tunnel. + +```sh +kubectl describe tcproxy -n temporal-system to-cluster-a +kubectl describe tcproxy -n temporal-system to-cluster-b +``` diff --git a/examples/cluster-proxy-mux/client.yaml b/examples/cluster-proxy-mux/client.yaml new file mode 100644 index 0000000..6a72a0a --- /dev/null +++ b/examples/cluster-proxy-mux/client.yaml @@ -0,0 +1,39 @@ +# Client-side TemporalClusterProxy: deployed next to cluster-a, dials out to the +# mux address published by cluster-b's server proxy. The operator deploys an +# s2s-proxy instance, mints a cert-manager TLS certificate signed by temporal-ca, +# and registers cluster-b as a remote cluster on cluster-a's frontend. +# +# Prerequisites: +# - cluster-a (TemporalCluster) in the same namespace +# - cert-manager installed with a temporal-ca Issuer in the same namespace +# - cluster-b's server proxy LoadBalancer address known (set in serverAddress) +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: to-cluster-b + namespace: temporal-system +spec: + # Reference to the local TemporalCluster this proxy fronts (cluster-a). + localClusterRef: + name: cluster-a + # Must match cluster-a's clusterMetadata.currentClusterName. + localClusterName: cluster-a + # The remote replication cluster on the far side of the mux. + peer: + name: cluster-b + mux: + # Client role: dials out to cluster-b's server proxy mux address. + role: client + client: + # Set this to the LoadBalancer address published by cluster-b's server proxy + # (see server.yaml). In a single-network setup use the in-cluster Service DNS: + # ..svc.cluster.local:6334 + serverAddress: cluster-b-proxy.example.com:6334 + tls: + provider: cert-manager + issuerRef: + name: temporal-ca + # Optional: supply cluster-b's CA separately when cluster-a and cluster-b + # use different cert-manager issuers. Omit when both use the same CA. + peerCARef: + name: cluster-b-mux-ca diff --git a/examples/cluster-proxy-mux/server.yaml b/examples/cluster-proxy-mux/server.yaml new file mode 100644 index 0000000..4918a82 --- /dev/null +++ b/examples/cluster-proxy-mux/server.yaml @@ -0,0 +1,36 @@ +# Server-side TemporalClusterProxy: deployed next to cluster-b, opens the mux +# port (6334) that cluster-a's client proxy dials into. The operator deploys an +# s2s-proxy instance, mints a cert-manager TLS certificate signed by temporal-ca, +# and registers cluster-a as a remote cluster on cluster-b's frontend. +# +# Prerequisites: +# - cluster-b (TemporalCluster) in the same namespace +# - cert-manager installed with a temporal-ca Issuer in the same namespace +# - cluster-a's client proxy configured to dial this service on port 6334 +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: to-cluster-a + namespace: temporal-system +spec: + # Reference to the local TemporalCluster this proxy fronts (cluster-b). + localClusterRef: + name: cluster-b + # Must match cluster-b's clusterMetadata.currentClusterName. + localClusterName: cluster-b + # The remote replication cluster on the far side of the mux. + peer: + name: cluster-a + mux: + # Server role: opens a listening mux port for cluster-a's client proxy to dial. + role: server + server: + listenPort: 6334 + # Expose the mux port via a LoadBalancer so cluster-a (in a different + # network) can dial in. Use ClusterIP when both clusters share one network. + exposure: + type: LoadBalancer + tls: + provider: cert-manager + issuerRef: + name: temporal-ca diff --git a/internal/controller/temporalclusterproxy_controller.go b/internal/controller/temporalclusterproxy_controller.go new file mode 100644 index 0000000..5014407 --- /dev/null +++ b/internal/controller/temporalclusterproxy_controller.go @@ -0,0 +1,320 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" + "github.com/bmorton/temporal-operator/internal/temporal" +) + +const ( + clusterProxyFinalizer = "temporal.bmor10.com/clusterproxy" + proxyServicesFieldOwner = client.FieldOwner("temporal-operator-clusterproxy") + proxyTLSProviderSecret = "secret" +) + +// TemporalClusterProxyReconciler deploys an s2s-proxy for one local cluster and +// registers the peer as a remote cluster via the local proxy. +type TemporalClusterProxyReconciler struct { + client.Client + Scheme *runtime.Scheme + ClientFactory temporal.RemoteClusterClientFactory +} + +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=temporal.bmor10.com,resources=temporalclusterproxies/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services;configmaps;secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete + +func (r *TemporalClusterProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var cr temporalv1alpha1.TemporalClusterProxy + if err := r.Get(ctx, req.NamespacedName, &cr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !cr.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.reconcileDelete(ctx, &cr) + } + if !controllerutil.ContainsFinalizer(&cr, clusterProxyFinalizer) { + controllerutil.AddFinalizer(&cr, clusterProxyFinalizer) + if err := r.Update(ctx, &cr); err != nil { + return ctrl.Result{}, err + } + } + + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err != nil { + if errors.Is(err, ErrTargetNotFound) { + r.setReady(&cr, metav1.ConditionFalse, temporalv1alpha1.ReasonClusterNotReady, "local cluster not found") + return ctrl.Result{RequeueAfter: namespaceDriftRequeue}, r.statusUpdate(ctx, &cr) + } + return ctrl.Result{}, err + } + + // Render + apply proxy resources. + if err := r.applyResources(ctx, &cr, target.Address); err != nil { + return ctrl.Result{}, err + } + + deployReady := r.deploymentAvailable(ctx, &cr) + r.setProxyDeployedCondition(&cr, deployReady) + + mtlsReady, mtlsMsg := r.checkMTLS(ctx, &cr) + r.setMTLSCondition(&cr, mtlsReady, mtlsMsg) + + // Publish endpoint for server role. + if cr.Spec.Mux.Role == temporalv1alpha1.ProxyRoleServer && cr.Spec.Mux.Server != nil { + cr.Status.ProxyEndpoint = r.serverEndpoint(ctx, &cr) + } + + // Register the peer via the local proxy once the proxy and local cluster are ready. + registered := r.reconcileRegistration(ctx, &cr, deployReady, target.Ready) + + switch { + case !mtlsReady: + r.setReady(&cr, metav1.ConditionFalse, temporalv1alpha1.ReasonMTLSNotReady, mtlsMsg) + case deployReady && registered: + r.setReady(&cr, metav1.ConditionTrue, temporalv1alpha1.ReasonProxyReady, "proxy deployed and peer registered") + default: + r.setReady(&cr, metav1.ConditionFalse, temporalv1alpha1.ReasonProxyNotReady, "proxy not fully converged") + } + return ctrl.Result{RequeueAfter: namespaceDriftRequeue}, r.statusUpdate(ctx, &cr) +} + +// checkMTLS reports whether the mux TLS material is present and complete. For a +// cert-manager provider the Secret appears once the Certificate is issued; for a +// BYO secret provider it must already exist with the certificate, key, and CA +// keys. It returns false with a diagnostic message while the material is absent +// or incomplete so a missing/misconfigured secret surfaces an explicit reason +// instead of a silently stuck Deployment. +func (r *TemporalClusterProxyReconciler) checkMTLS(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) (bool, string) { + secretName := resources.ClusterProxyTLSSecretName(cr) + var sec corev1.Secret + if err := r.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: secretName}, &sec); err != nil { + if apierrors.IsNotFound(err) { + if cr.Spec.Mux.TLS.Provider == proxyTLSProviderSecret { + return false, fmt.Sprintf("mux TLS secret %q not found", secretName) + } + return false, fmt.Sprintf("mux TLS certificate secret %q not yet issued", secretName) + } + return false, fmt.Sprintf("reading mux TLS secret %q: %v", secretName, err) + } + + certKey, keyKey, caKey := "tls.crt", "tls.key", "ca.crt" + if cr.Spec.Mux.TLS.Provider == proxyTLSProviderSecret && cr.Spec.Mux.TLS.SecretRef != nil { + ref := cr.Spec.Mux.TLS.SecretRef + if ref.CertKey != "" { + certKey = ref.CertKey + } + if ref.KeyKey != "" { + keyKey = ref.KeyKey + } + if ref.CAKey != "" { + caKey = ref.CAKey + } + } + for _, k := range []string{certKey, keyKey, caKey} { + if len(sec.Data[k]) == 0 { + return false, fmt.Sprintf("mux TLS secret %q is missing key %q", secretName, k) + } + } + return true, "mux TLS material present" +} + +func (r *TemporalClusterProxyReconciler) setMTLSCondition(cr *temporalv1alpha1.TemporalClusterProxy, ready bool, message string) { + status := metav1.ConditionFalse + reason := temporalv1alpha1.ReasonMTLSNotReady + if ready { + status = metav1.ConditionTrue + reason = temporalv1alpha1.ReasonProxyReady + } + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: temporalv1alpha1.ConditionMTLSReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: cr.Generation, + }) +} + +// applyResources renders and server-side-applies the proxy's owned resources. +func (r *TemporalClusterProxyReconciler) applyResources(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy, localFrontendAddress string) error { + configYAML, err := resources.BuildClusterProxyConfig(cr, localFrontendAddress) + if err != nil { + return err + } + sum := sha256.Sum256([]byte(configYAML)) + configHash := hex.EncodeToString(sum[:]) + + objs := []client.Object{ + resources.BuildClusterProxyConfigMap(cr, configYAML), + resources.BuildClusterProxyService(cr), + resources.BuildClusterProxyDeployment(cr, configHash), + } + if cr.Spec.Mux.TLS.Provider != proxyTLSProviderSecret && cr.Spec.Mux.TLS.IssuerRef != nil { + objs = append([]client.Object{resources.BuildClusterProxyCertificate(cr)}, objs...) + } + for _, obj := range objs { + if err := r.apply(ctx, cr, obj); err != nil { + return err + } + } + return nil +} + +func (r *TemporalClusterProxyReconciler) setProxyDeployedCondition(cr *temporalv1alpha1.TemporalClusterProxy, deployReady bool) { + if deployReady { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionProxyDeployed, Status: metav1.ConditionTrue, Reason: temporalv1alpha1.ReasonProxyReady, Message: "proxy deployment available", ObservedGeneration: cr.Generation}) + return + } + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionProxyDeployed, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonProxyNotReady, Message: "proxy deployment not yet available", ObservedGeneration: cr.Generation}) +} + +// reconcileRegistration registers the peer via the local proxy when both the +// proxy Deployment and the local cluster are ready, and records the result as a +// condition. It returns whether the peer is registered. +func (r *TemporalClusterProxyReconciler) reconcileRegistration(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy, deployReady, clusterReady bool) bool { + if !deployReady || !clusterReady { + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonClusterNotReady, Message: "waiting for proxy and local cluster", ObservedGeneration: cr.Generation}) + return false + } + if err := r.registerPeer(ctx, cr); err != nil { + logf.FromContext(ctx).Error(err, "registering peer via proxy") + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionFalse, Reason: temporalv1alpha1.ReasonRegistrationFailed, Message: err.Error(), ObservedGeneration: cr.Generation}) + return false + } + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{Type: temporalv1alpha1.ConditionRemoteClusterRegistered, Status: metav1.ConditionTrue, Reason: temporalv1alpha1.ReasonProxyReady, Message: "peer registered via local proxy", ObservedGeneration: cr.Generation}) + return true +} + +// registerPeer dials the local Temporal frontend and registers the peer with the +// local proxy tcpServer address as its frontend. +func (r *TemporalClusterProxyReconciler) registerPeer(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err != nil { + return err + } + c, err := r.clientFactory()(ctx, target.Address, target.TLSConfig) + if err != nil { + return err + } + defer func() { _ = c.Close() }() + proxyAddr := fmt.Sprintf("%s.%s.svc.cluster.local:%d", resources.ClusterProxyServiceName(cr), cr.Namespace, resources.ProxyTCPServerPort) + enable := cr.Spec.Peer.EnableConnection == nil || *cr.Spec.Peer.EnableConnection + return c.UpsertRemoteCluster(ctx, proxyAddr, enable) +} + +func (r *TemporalClusterProxyReconciler) reconcileDelete(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + if !controllerutil.ContainsFinalizer(cr, clusterProxyFinalizer) { + return nil + } + target, err := resolveTarget(ctx, r.Client, cr.Namespace, cr.Spec.LocalClusterRef) + if err == nil && target.Ready { + if c, derr := r.clientFactory()(ctx, target.Address, target.TLSConfig); derr == nil { + _ = c.RemoveRemoteCluster(ctx, cr.Spec.Peer.Name) + _ = c.Close() + } + } + // Owned resources are GC'd via owner references. + controllerutil.RemoveFinalizer(cr, clusterProxyFinalizer) + return r.Update(ctx, cr) +} + +func (r *TemporalClusterProxyReconciler) deploymentAvailable(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) bool { + var dep appsv1.Deployment + key := types.NamespacedName{Namespace: cr.Namespace, Name: resources.ClusterProxyName(cr)} + if err := r.Get(ctx, key, &dep); err != nil { + return false + } + return dep.Status.AvailableReplicas > 0 +} + +func (r *TemporalClusterProxyReconciler) serverEndpoint(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) string { + // For LoadBalancer, surface the assigned ingress; otherwise the in-cluster DNS name. + var svc corev1.Service + if err := r.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: resources.ClusterProxyServiceName(cr)}, &svc); err == nil { + for _, ing := range svc.Status.LoadBalancer.Ingress { + host := ing.Hostname + if host == "" { + host = ing.IP + } + if host != "" { + return fmt.Sprintf("%s:%d", host, cr.Spec.Mux.Server.ListenPort) + } + } + } + return fmt.Sprintf("%s.%s.svc.cluster.local:%d", resources.ClusterProxyServiceName(cr), cr.Namespace, cr.Spec.Mux.Server.ListenPort) +} + +func (r *TemporalClusterProxyReconciler) apply(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy, obj client.Object) error { + if err := controllerutil.SetControllerReference(cr, obj, r.Scheme); err != nil { + return err + } + return serverSideApply(ctx, r.Client, r.Scheme, obj, proxyServicesFieldOwner) +} + +func (r *TemporalClusterProxyReconciler) clientFactory() temporal.RemoteClusterClientFactory { + if r.ClientFactory != nil { + return r.ClientFactory + } + return temporal.NewRemoteClusterClient +} + +func (r *TemporalClusterProxyReconciler) setReady(cr *temporalv1alpha1.TemporalClusterProxy, status metav1.ConditionStatus, reason, message string) { + cr.Status.ObservedGeneration = cr.Generation + meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: temporalv1alpha1.ConditionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: cr.Generation, + }) +} + +func (r *TemporalClusterProxyReconciler) statusUpdate(ctx context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + return client.IgnoreNotFound(r.Status().Update(ctx, cr)) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TemporalClusterProxyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&temporalv1alpha1.TemporalClusterProxy{}). + Owns(&appsv1.Deployment{}). + Named("temporalclusterproxy"). + Complete(r) +} diff --git a/internal/controller/temporalclusterproxy_controller_test.go b/internal/controller/temporalclusterproxy_controller_test.go new file mode 100644 index 0000000..3cb3266 --- /dev/null +++ b/internal/controller/temporalclusterproxy_controller_test.go @@ -0,0 +1,240 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/tls" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" + "github.com/bmorton/temporal-operator/internal/temporal" +) + +var _ = Describe("TemporalClusterProxy reconciler", func() { + ctx := context.Background() + var counter int + var fakes map[string]*fakeRemoteClient // key: dialed address + + var factory temporal.RemoteClusterClientFactory = func(_ context.Context, address string, _ *tls.Config) (temporal.RemoteClusterClient, error) { + f := fakes[address] + if f == nil { + f = &fakeRemoteClient{view: map[string]temporal.RemoteClusterInfo{}, addrName: map[string]string{}} + fakes[address] = f + } + return f, nil + } + + reconcileProxy := func(name string) { + r := &TemporalClusterProxyReconciler{Client: k8sClient, Scheme: k8sClient.Scheme(), ClientFactory: factory} + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: name, Namespace: "default"}}) + Expect(err).NotTo(HaveOccurred()) + } + + // readyProxyCluster creates a Ready TemporalCluster, returning its k8s name. + readyProxyCluster := func(clusterName string) string { + counter++ + name := fmt.Sprintf("proxy-cluster-%d", counter) + spec := validClusterSpec("1.31.1") + spec.ClusterMetadata = &temporalv1alpha1.ClusterMetadataSpec{ + EnableGlobalNamespace: true, + CurrentClusterName: clusterName, + } + c := &temporalv1alpha1.TemporalCluster{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: spec, + } + Expect(k8sClient.Create(ctx, c)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, c) }) + meta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ + Type: temporalv1alpha1.ConditionReady, Status: metav1.ConditionTrue, Reason: "Ready", Message: "ready", + }) + Expect(k8sClient.Status().Update(ctx, c)).To(Succeed()) + return name + } + + BeforeEach(func() { + fakes = map[string]*fakeRemoteClient{} + }) + + It("deploys the proxy and sets conditions", func() { + localCluster := readyProxyCluster("cluster-a") + + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "mux-tls", Namespace: "default"}, + Data: map[string][]byte{"tls.crt": []byte("x"), "tls.key": []byte("y"), "ca.crt": []byte("z")}, + } + Expect(k8sClient.Create(ctx, tlsSecret)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, tlsSecret) }) + + proxyName := fmt.Sprintf("proxy-%d", counter) + proxy := &temporalv1alpha1.TemporalClusterProxy{ + ObjectMeta: metav1.ObjectMeta{Name: proxyName, Namespace: "default"}, + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: localCluster}, + Peer: temporalv1alpha1.ProxyPeer{ + Name: "cluster-b", + }, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ListenPort: 6334}, + TLS: temporalv1alpha1.ProxyMuxTLS{ + Provider: "secret", + SecretRef: &temporalv1alpha1.SecretReference{Name: "mux-tls"}, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, proxy)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, proxy) }) + + reconcileProxy(proxyName) // adds finalizer + reconcileProxy(proxyName) // renders + applies resources + + got := &temporalv1alpha1.TemporalClusterProxy{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: proxyName, Namespace: "default"}, got)).To(Succeed()) + + // Deployment, ConfigMap and Service exist, owned by the proxy CR. + var dep appsv1.Deployment + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resources.ClusterProxyName(got), Namespace: "default"}, &dep)).To(Succeed()) + Expect(dep.OwnerReferences).To(ContainElement(HaveField("UID", got.UID))) + + var cm corev1.ConfigMap + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resources.ClusterProxyConfigMapName(got), Namespace: "default"}, &cm)).To(Succeed()) + Expect(cm.OwnerReferences).To(ContainElement(HaveField("UID", got.UID))) + + var svc corev1.Service + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resources.ClusterProxyServiceName(got), Namespace: "default"}, &svc)).To(Succeed()) + Expect(svc.OwnerReferences).To(ContainElement(HaveField("UID", got.UID))) + + // The ProxyDeployed condition is present. + cond := meta.FindStatusCondition(got.Status.Conditions, temporalv1alpha1.ConditionProxyDeployed) + Expect(cond).NotTo(BeNil()) + + // The BYO TLS secret is present and complete, so MTLSReady is True. + Expect(meta.IsStatusConditionTrue(got.Status.Conditions, temporalv1alpha1.ConditionMTLSReady)).To(BeTrue()) + }) + + It("reports MTLSReady=False and Ready=False when the BYO TLS secret is missing", func() { + localCluster := readyProxyCluster("cluster-a") + + proxyName := fmt.Sprintf("proxy-nomtls-%d", counter) + proxy := &temporalv1alpha1.TemporalClusterProxy{ + ObjectMeta: metav1.ObjectMeta{Name: proxyName, Namespace: "default"}, + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: localCluster}, + Peer: temporalv1alpha1.ProxyPeer{Name: "cluster-b"}, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ListenPort: 6334}, + TLS: temporalv1alpha1.ProxyMuxTLS{ + Provider: "secret", + SecretRef: &temporalv1alpha1.SecretReference{Name: "does-not-exist"}, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, proxy)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, proxy) }) + + reconcileProxy(proxyName) // adds finalizer + reconcileProxy(proxyName) // renders + applies, checks MTLS + + got := &temporalv1alpha1.TemporalClusterProxy{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: proxyName, Namespace: "default"}, got)).To(Succeed()) + + mtls := meta.FindStatusCondition(got.Status.Conditions, temporalv1alpha1.ConditionMTLSReady) + Expect(mtls).NotTo(BeNil()) + Expect(mtls.Status).To(Equal(metav1.ConditionFalse)) + Expect(meta.IsStatusConditionTrue(got.Status.Conditions, temporalv1alpha1.ConditionReady)).To(BeFalse()) + }) + + It("registers the local proxy address with the local cluster when the deployment is available", func() { + localCluster := readyProxyCluster("cluster-a") + + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "mux-tls-reg", Namespace: "default"}, + Data: map[string][]byte{"tls.crt": []byte("x"), "tls.key": []byte("y"), "ca.crt": []byte("z")}, + } + Expect(k8sClient.Create(ctx, tlsSecret)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, tlsSecret) }) + + proxyName := fmt.Sprintf("proxy-reg-%d", counter) + proxy := &temporalv1alpha1.TemporalClusterProxy{ + ObjectMeta: metav1.ObjectMeta{Name: proxyName, Namespace: "default"}, + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: localCluster}, + Peer: temporalv1alpha1.ProxyPeer{Name: "cluster-b"}, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ListenPort: 6334}, + TLS: temporalv1alpha1.ProxyMuxTLS{ + Provider: "secret", + SecretRef: &temporalv1alpha1.SecretReference{Name: "mux-tls-reg"}, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, proxy)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, proxy) }) + + reconcileProxy(proxyName) // adds finalizer + reconcileProxy(proxyName) // renders + applies resources; Deployment is created + + // Patch the Deployment status to simulate AvailableReplicas=1. + // envtest has no kubelet, so we must do this manually. + // availableReplicas cannot exceed readyReplicas; set both. + var dep appsv1.Deployment + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resources.ClusterProxyName(proxy), Namespace: "default"}, &dep)).To(Succeed()) + dep.Status.Replicas = 1 + dep.Status.ReadyReplicas = 1 + dep.Status.AvailableReplicas = 1 + Expect(k8sClient.Status().Update(ctx, &dep)).To(Succeed()) + + // Reconcile again: deploymentAvailable→true, clusterReady→true → registerPeer fires. + reconcileProxy(proxyName) + + // Determine the local cluster frontend address (the key the factory uses). + localC := &temporalv1alpha1.TemporalCluster{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: localCluster, Namespace: "default"}, localC)).To(Succeed()) + localAddr := frontendAddress(localC) + + // Expected proxy DNS address that must be upserted on the local cluster. + expectedProxyAddr := fmt.Sprintf("%s.%s.svc.cluster.local:%d", + resources.ClusterProxyServiceName(proxy), "default", resources.ProxyTCPServerPort) + + Expect(fakes).To(HaveKey(localAddr), "factory should have dialed the local cluster frontend") + Expect(fakes[localAddr].upserts).To(ContainElement(expectedProxyAddr), + "UpsertRemoteCluster must be called with the proxy's in-cluster DNS address") + + // Verify the CR's RemoteClusterRegistered and Ready conditions are True. + got := &temporalv1alpha1.TemporalClusterProxy{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: proxyName, Namespace: "default"}, got)).To(Succeed()) + Expect(meta.IsStatusConditionTrue(got.Status.Conditions, temporalv1alpha1.ConditionRemoteClusterRegistered)).To(BeTrue()) + Expect(meta.IsStatusConditionTrue(got.Status.Conditions, temporalv1alpha1.ConditionReady)).To(BeTrue()) + }) +}) diff --git a/internal/resources/clusterproxy.go b/internal/resources/clusterproxy.go new file mode 100644 index 0000000..8a7bf57 --- /dev/null +++ b/internal/resources/clusterproxy.go @@ -0,0 +1,184 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +// DefaultClusterProxyImage is the pinned s2s-proxy image. +const DefaultClusterProxyImage = "temporalio/s2s-proxy:v0.2.1" + +// ClusterProxyName returns the proxy Deployment (and base) name. +func ClusterProxyName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return cr.Name + "-s2s-proxy" +} + +// ClusterProxyConfigMapName returns the rendered-config ConfigMap name. +func ClusterProxyConfigMapName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) + "-config" +} + +// ClusterProxyServiceName returns the proxy Service name. +func ClusterProxyServiceName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) +} + +// ClusterProxyCertName returns the cert-manager Certificate name. +func ClusterProxyCertName(cr *temporalv1alpha1.TemporalClusterProxy) string { + return ClusterProxyName(cr) + "-tls" +} + +// ClusterProxyTLSSecretName returns the mux TLS Secret name (own material). +func ClusterProxyTLSSecretName(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Mux.TLS.Provider == "secret" && cr.Spec.Mux.TLS.SecretRef != nil { + return cr.Spec.Mux.TLS.SecretRef.Name + } + return ClusterProxyCertName(cr) +} + +// ClusterProxyLabels returns the standard label set for proxy resources. +func ClusterProxyLabels(cr *temporalv1alpha1.TemporalClusterProxy) map[string]string { + return map[string]string{ + LabelName: "s2s-proxy", + LabelInstance: cr.Name, + LabelComponent: "s2s-proxy", + LabelManagedBy: managedByValue, + } +} + +func clusterProxyImage(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Image != "" { + return cr.Spec.Image + } + return DefaultClusterProxyImage +} + +// BuildClusterProxyConfigMap wraps the rendered config YAML in a ConfigMap. +func BuildClusterProxyConfigMap(cr *temporalv1alpha1.TemporalClusterProxy, configYAML string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyConfigMapName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr)}, + Data: map[string]string{ProxyConfigFileName: configYAML}, + } +} + +// BuildClusterProxyCertificate builds the mux mTLS Certificate. Only call when +// mux.tls.provider is cert-manager (IssuerRef set). +func BuildClusterProxyCertificate(cr *temporalv1alpha1.TemporalClusterProxy) *certmanagerv1.Certificate { + var dnsNames []string + svc := ClusterProxyServiceName(cr) + dnsNames = append(dnsNames, + svc, + svc+"."+cr.Namespace+".svc", + svc+"."+cr.Namespace+".svc.cluster.local", + ) + return &certmanagerv1.Certificate{ + TypeMeta: metav1.TypeMeta{APIVersion: "cert-manager.io/v1", Kind: "Certificate"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyCertName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr)}, + Spec: certmanagerv1.CertificateSpec{ + SecretName: ClusterProxyTLSSecretName(cr), + CommonName: ClusterProxyName(cr), + DNSNames: dnsNames, + IssuerRef: issuerRef(cr.Spec.Mux.TLS.IssuerRef), + Usages: []certmanagerv1.KeyUsage{certmanagerv1.UsageServerAuth, certmanagerv1.UsageClientAuth}, + }, + } +} + +// BuildClusterProxyService builds the proxy Service. It always exposes the +// tcpServer port (ClusterIP, for the local Temporal) and, for the server role, +// the mux listen port using the configured exposure. +func BuildClusterProxyService(cr *temporalv1alpha1.TemporalClusterProxy) *corev1.Service { + ports := []corev1.ServicePort{ + {Name: "tcp-server", Port: ProxyTCPServerPort, TargetPort: intstrFromInt(ProxyTCPServerPort)}, + } + svcType := corev1.ServiceTypeClusterIP + var annotations map[string]string + if cr.Spec.Mux.Role == temporalv1alpha1.ProxyRoleServer && cr.Spec.Mux.Server != nil { + ports = append(ports, corev1.ServicePort{ + Name: "mux", Port: cr.Spec.Mux.Server.ListenPort, TargetPort: intstrFromInt(cr.Spec.Mux.Server.ListenPort), + }) + if ex := cr.Spec.Mux.Server.Exposure; ex != nil { + if ex.Type != "" { + svcType = ex.Type + } + annotations = ex.Annotations + } + } + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyServiceName(cr), Namespace: cr.Namespace, Labels: ClusterProxyLabels(cr), Annotations: annotations}, + Spec: corev1.ServiceSpec{ + Type: svcType, + Selector: ClusterProxyLabels(cr), + Ports: ports, + }, + } +} + +// BuildClusterProxyDeployment builds the s2s-proxy Deployment. configHash stamps +// the pod template so config/cert changes trigger a rollout. +func BuildClusterProxyDeployment(cr *temporalv1alpha1.TemporalClusterProxy, configHash string) *appsv1.Deployment { + volumes := []corev1.Volume{ + {Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: ClusterProxyConfigMapName(cr)}, + }}}, + {Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: ClusterProxyTLSSecretName(cr), + }}}, + } + mounts := []corev1.VolumeMount{ + {Name: "config", MountPath: ProxyConfigMountPath, ReadOnly: true}, + {Name: "tls", MountPath: ProxyTLSMountPath, ReadOnly: true}, + } + if ref := cr.Spec.Mux.TLS.PeerCARef; ref != nil { + volumes = append(volumes, corev1.Volume{Name: "peer-ca", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: ref.Name}}}) + mounts = append(mounts, corev1.VolumeMount{Name: "peer-ca", MountPath: ProxyPeerCAMountPath, ReadOnly: true}) + } + + replicas := int32(1) + labels := ClusterProxyLabels(cr) + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Name: ClusterProxyName(cr), Namespace: cr.Namespace, Labels: labels}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{ConfigHashAnnotation: configHash}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "s2s-proxy", + Image: clusterProxyImage(cr), + Env: []corev1.EnvVar{{Name: "CONFIG_YML", Value: ProxyConfigMountPath + "/" + ProxyConfigFileName}}, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } +} diff --git a/internal/resources/clusterproxy_test.go b/internal/resources/clusterproxy_test.go new file mode 100644 index 0000000..a8525af --- /dev/null +++ b/internal/resources/clusterproxy_test.go @@ -0,0 +1,323 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "strings" + "testing" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + sigsyaml "sigs.k8s.io/yaml" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" + "github.com/bmorton/temporal-operator/internal/resources" +) + +func serverProxyCR() *temporalv1alpha1.TemporalClusterProxy { + enable := true + return &temporalv1alpha1.TemporalClusterProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "link", Namespace: "temporal-system"}, + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: "cluster-a"}, + LocalClusterName: "cluster-a", + Peer: temporalv1alpha1.ProxyPeer{Name: "cluster-b", EnableConnection: &enable}, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ListenPort: 6334}, + TLS: temporalv1alpha1.ProxyMuxTLS{Provider: "cert-manager"}, + }, + }, + } +} + +func TestBuildClusterProxyConfig_ServerRole(t *testing.T) { + out, err := resources.BuildClusterProxyConfig(serverProxyCR(), "cluster-a-frontend.temporal-system.svc.cluster.local:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + + var cfg struct { + ClusterConnections []struct { + Name string `json:"name"` + Local struct { + ConnectionType string `json:"connectionType"` + TCPClient struct{ Address string } `json:"tcpClient"` + TCPServer struct{ Address string } `json:"tcpServer"` + } `json:"local"` + Remote struct { + ConnectionType string `json:"connectionType"` + MuxAddressInfo struct { + Address string `json:"address"` + TLS struct { + CertificatePath string `json:"certificatePath"` + KeyPath string `json:"keyPath"` + RemoteCAPath string `json:"remoteCAPath"` + } `json:"tls"` + } `json:"muxAddressInfo"` + } `json:"remote"` + } `json:"clusterConnections"` + } + if err := sigsyaml.Unmarshal([]byte(out), &cfg); err != nil { + t.Fatalf("unmarshal rendered config: %v\n%s", err, out) + } + if len(cfg.ClusterConnections) != 1 { + t.Fatalf("want 1 connection, got %d", len(cfg.ClusterConnections)) + } + c := cfg.ClusterConnections[0] + if c.Local.ConnectionType != "tcp" { + t.Errorf("local.connectionType = %q, want tcp", c.Local.ConnectionType) + } + if c.Local.TCPClient.Address != "cluster-a-frontend.temporal-system.svc.cluster.local:7233" { + t.Errorf("tcpClient.address = %q", c.Local.TCPClient.Address) + } + if !strings.HasSuffix(c.Local.TCPServer.Address, "6233") { + t.Errorf("tcpServer.address = %q, want :6233", c.Local.TCPServer.Address) + } + if c.Remote.ConnectionType != "mux-server" { + t.Errorf("remote.connectionType = %q, want mux-server", c.Remote.ConnectionType) + } + if !strings.HasSuffix(c.Remote.MuxAddressInfo.Address, "6334") { + t.Errorf("mux address = %q, want :6334", c.Remote.MuxAddressInfo.Address) + } + if c.Remote.MuxAddressInfo.TLS.CertificatePath != resources.ProxyTLSMountPath+"/tls.crt" { + t.Errorf("certificatePath = %q", c.Remote.MuxAddressInfo.TLS.CertificatePath) + } + if c.Remote.MuxAddressInfo.TLS.RemoteCAPath != resources.ProxyTLSMountPath+"/ca.crt" { + t.Errorf("remoteCAPath = %q (want own ca.crt when no peerCARef)", c.Remote.MuxAddressInfo.TLS.RemoteCAPath) + } +} + +func TestBuildClusterProxyConfig_ClientRoleWithTranslation(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + cr.Spec.Mux.Server = nil + cr.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "b.example.com:6334"} + cr.Spec.Translation = &temporalv1alpha1.ProxyTranslation{ + Namespaces: []temporalv1alpha1.ProxyNamespaceMapping{{Local: "ns", Remote: "ns.acct"}}, + } + + out, err := resources.BuildClusterProxyConfig(cr, "cluster-a-frontend:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(out, "mux-client") { + t.Errorf("expected mux-client in:\n%s", out) + } + if !strings.Contains(out, "b.example.com:6334") { + t.Errorf("expected serverAddress in:\n%s", out) + } + if !strings.Contains(out, "ns.acct") { + t.Errorf("expected namespace translation in:\n%s", out) + } + // s2s-proxy requires the mux-client to verify the server certificate by name; + // with no explicit override it is derived from the server address host. + if !strings.Contains(out, "caServerName: b.example.com") { + t.Errorf("expected derived caServerName (host of serverAddress) in:\n%s", out) + } +} + +func TestBuildClusterProxyConfig_ClientCAServerNameOverrideAndSkip(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + cr.Spec.Mux.Server = nil + cr.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "b.example.com:6334"} + + // Explicit override wins over the derived host. + cr.Spec.Mux.TLS.CAServerName = "custom-name" + out, err := resources.BuildClusterProxyConfig(cr, "cluster-a-frontend:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(out, "caServerName: custom-name") { + t.Errorf("expected caServerName override in:\n%s", out) + } + + // SkipCAVerification true emits skipCAVerification and drops caServerName. + skip := true + cr.Spec.Mux.TLS.CAServerName = "" + cr.Spec.Mux.TLS.SkipCAVerification = &skip + out, err = resources.BuildClusterProxyConfig(cr, "cluster-a-frontend:7233") + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(out, "skipCAVerification: true") { + t.Errorf("expected skipCAVerification true in:\n%s", out) + } + if strings.Contains(out, "caServerName:") { + t.Errorf("caServerName must be omitted when skipping verification:\n%s", out) + } +} + +func TestBuildClusterProxyService_ServerExposesMux(t *testing.T) { + cr := serverProxyCR() + svc := resources.BuildClusterProxyService(cr) + if svc.Name != resources.ClusterProxyServiceName(cr) { + t.Errorf("service name = %q", svc.Name) + } + var haveTCP, haveMux bool + for _, p := range svc.Spec.Ports { + if p.Port == resources.ProxyTCPServerPort { + haveTCP = true + } + if p.Port == 6334 { + haveMux = true + } + } + if !haveTCP { + t.Error("expected tcpServer port 6233") + } + if !haveMux { + t.Error("expected mux port 6334 for server role") + } +} + +func TestBuildClusterProxyService_ClientOmitsMuxPort(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + cr.Spec.Mux.Server = nil + cr.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "b:6334"} + svc := resources.BuildClusterProxyService(cr) + for _, p := range svc.Spec.Ports { + if p.Name == "mux" { + t.Error("client role must not expose a mux port") + } + } +} + +func TestBuildClusterProxyDeployment_MountsConfigAndTLS(t *testing.T) { + cr := serverProxyCR() + dep := resources.BuildClusterProxyDeployment(cr, "abc123") + if dep.Name != resources.ClusterProxyName(cr) { + t.Errorf("deployment name = %q", dep.Name) + } + c := dep.Spec.Template.Spec.Containers[0] + var haveConfig, haveTLS bool + for _, m := range c.VolumeMounts { + if m.MountPath == resources.ProxyConfigMountPath { + haveConfig = true + } + if m.MountPath == resources.ProxyTLSMountPath { + haveTLS = true + } + } + if !haveConfig || !haveTLS { + t.Errorf("missing mounts: config=%v tls=%v", haveConfig, haveTLS) + } + if dep.Spec.Template.Annotations[resources.ConfigHashAnnotation] != "abc123" { + t.Errorf("config hash annotation = %q", dep.Spec.Template.Annotations[resources.ConfigHashAnnotation]) + } +} + +func TestBuildClusterProxyCertificate_UsesIssuer(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.TLS.IssuerRef = &temporalv1alpha1.IssuerReference{Name: "ca-issuer"} + crt := resources.BuildClusterProxyCertificate(cr) + if crt.Spec.IssuerRef.Name != "ca-issuer" { + t.Errorf("issuer = %q", crt.Spec.IssuerRef.Name) + } + if crt.Spec.SecretName != resources.ClusterProxyTLSSecretName(cr) { + t.Errorf("secretName = %q", crt.Spec.SecretName) + } +} + +func TestBuildClusterProxyDeployment_ConfigYMLEnvVar(t *testing.T) { + cr := serverProxyCR() + dep := resources.BuildClusterProxyDeployment(cr, "abc123") + c := dep.Spec.Template.Spec.Containers[0] + want := resources.ProxyConfigMountPath + "/" + resources.ProxyConfigFileName + for _, e := range c.Env { + if e.Name == "CONFIG_YML" { + if e.Value != want { + t.Errorf("CONFIG_YML = %q, want %q", e.Value, want) + } + return + } + } + t.Error("CONFIG_YML env var not found in container env") +} + +func TestBuildClusterProxyService_LoadBalancerExposure(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.Server.Exposure = &temporalv1alpha1.ServiceExposureSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Annotations: map[string]string{"k": "v"}, + } + svc := resources.BuildClusterProxyService(cr) + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + t.Errorf("service type = %q, want LoadBalancer", svc.Spec.Type) + } + if svc.Annotations["k"] != "v" { + t.Errorf("annotation k = %q, want v", svc.Annotations["k"]) + } +} + +func TestClusterProxyTLSSecretName_BYO(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.TLS = temporalv1alpha1.ProxyMuxTLS{ + Provider: "secret", + SecretRef: &temporalv1alpha1.SecretReference{Name: "byo-tls"}, + } + if got := resources.ClusterProxyTLSSecretName(cr); got != "byo-tls" { + t.Errorf("TLSSecretName (secret provider) = %q, want byo-tls", got) + } + + cr2 := serverProxyCR() + if got := resources.ClusterProxyTLSSecretName(cr2); got != resources.ClusterProxyCertName(cr2) { + t.Errorf("TLSSecretName (cert-manager) = %q, want %q", got, resources.ClusterProxyCertName(cr2)) + } +} + +func TestBuildClusterProxyCertificate_DNSNamesAndUsages(t *testing.T) { + cr := serverProxyCR() + cr.Spec.Mux.TLS.IssuerRef = &temporalv1alpha1.IssuerReference{Name: "ca-issuer"} + crt := resources.BuildClusterProxyCertificate(cr) + + svc := resources.ClusterProxyServiceName(cr) + ns := cr.Namespace + wantDNS := []string{ + svc, + svc + "." + ns + ".svc", + svc + "." + ns + ".svc.cluster.local", + } + dnsSet := make(map[string]bool, len(crt.Spec.DNSNames)) + for _, d := range crt.Spec.DNSNames { + dnsSet[d] = true + } + for _, d := range wantDNS { + if !dnsSet[d] { + t.Errorf("DNS name %q missing from certificate", d) + } + } + + var haveServer, haveClient bool + for _, u := range crt.Spec.Usages { + if u == certmanagerv1.UsageServerAuth { + haveServer = true + } + if u == certmanagerv1.UsageClientAuth { + haveClient = true + } + } + if !haveServer { + t.Error("UsageServerAuth missing from certificate usages") + } + if !haveClient { + t.Error("UsageClientAuth missing from certificate usages") + } +} diff --git a/internal/resources/clusterproxyconfig.go b/internal/resources/clusterproxyconfig.go new file mode 100644 index 0000000..4e177e8 --- /dev/null +++ b/internal/resources/clusterproxyconfig.go @@ -0,0 +1,251 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + "net" + + sigsyaml "sigs.k8s.io/yaml" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +// Mount paths and ports for the rendered s2s-proxy pod. +const ( + ProxyConfigMountPath = "/etc/s2s-proxy" + ProxyConfigFileName = "config.yaml" + ProxyTLSMountPath = "/etc/s2s-proxy/tls" + ProxyPeerCAMountPath = "/etc/s2s-proxy/peer-ca" + + // ProxyTCPServerPort is the port the proxy exposes for the local Temporal to + // reach as the peer's frontend address. + ProxyTCPServerPort int32 = 6233 +) + +// DefaultAllowedAdminMethods is the standard admin-service allowlist required +// for cross-cluster replication. +func DefaultAllowedAdminMethods() []string { + return []string{ + "AddOrUpdateRemoteCluster", + "DescribeCluster", + "DescribeMutableState", + "GetNamespaceReplicationMessages", + "GetWorkflowExecutionRawHistoryV2", + "ListClusters", + "StreamWorkflowReplicationMessages", + } +} + +// --- s2s-proxy config schema (subset we render) --- + +type proxyTLS struct { + CertificatePath string `json:"certificatePath"` + KeyPath string `json:"keyPath"` + RemoteCAPath string `json:"remoteCAPath"` + CAServerName string `json:"caServerName,omitempty"` + SkipCAVerification bool `json:"skipCAVerification,omitempty"` +} + +type proxyAddressInfo struct { + Address string `json:"address"` + TLS proxyTLS `json:"tls"` +} + +type proxyTCPEndpoint struct { + Address string `json:"address"` +} + +type proxyLocal struct { + ConnectionType string `json:"connectionType"` + TCPClient proxyTCPEndpoint `json:"tcpClient"` + TCPServer proxyTCPEndpoint `json:"tcpServer"` +} + +type proxyRemote struct { + ConnectionType string `json:"connectionType"` + MuxCount *int32 `json:"muxCount,omitempty"` + MuxAddressInfo proxyAddressInfo `json:"muxAddressInfo"` +} + +type proxyNamespaceMapping struct { + Local string `json:"local"` + Remote string `json:"remote"` +} + +type proxyNamespaceTranslation struct { + Mappings []proxyNamespaceMapping `json:"mappings"` +} + +type proxyFieldMapping struct { + LocalFieldName string `json:"localFieldName"` + RemoteFieldName string `json:"remoteFieldName"` +} + +type proxySANamespaceMapping struct { + Name string `json:"name"` + NamespaceID string `json:"namespaceId"` + Mappings []proxyFieldMapping `json:"mappings"` +} + +type proxySATranslation struct { + NamespaceMappings []proxySANamespaceMapping `json:"namespaceMappings"` +} + +type proxyFailover struct { + Local int64 `json:"local"` + Remote int64 `json:"remote"` +} + +type proxyACLPolicy struct { + AllowedMethods map[string][]string `json:"allowedMethods"` + AllowedNamespaces []string `json:"allowedNamespaces,omitempty"` +} + +type proxyConnection struct { + Name string `json:"name"` + Local proxyLocal `json:"local"` + Remote proxyRemote `json:"remote"` + NamespaceTranslation *proxyNamespaceTranslation `json:"namespaceTranslation,omitempty"` + SearchAttributeTranslation *proxySATranslation `json:"searchAttributeTranslation,omitempty"` + FailoverVersionIncrementTranslation *proxyFailover `json:"failoverVersionIncrementTranslation,omitempty"` + ACLPolicy *proxyACLPolicy `json:"aclPolicy,omitempty"` +} + +type proxyConfigFile struct { + ClusterConnections []proxyConnection `json:"clusterConnections"` +} + +// BuildClusterProxyConfig renders the s2s-proxy config YAML for one CR. It is +// pure: localFrontendAddress is resolved by the caller. +func BuildClusterProxyConfig(cr *temporalv1alpha1.TemporalClusterProxy, localFrontendAddress string) (string, error) { + mux := cr.Spec.Mux + + skipVerify := mux.TLS.SkipCAVerification != nil && *mux.TLS.SkipCAVerification + remote := proxyRemote{ + MuxCount: mux.MuxCount, + MuxAddressInfo: proxyAddressInfo{ + TLS: proxyTLS{ + CertificatePath: ProxyTLSMountPath + "/tls.crt", + KeyPath: ProxyTLSMountPath + "/tls.key", + RemoteCAPath: proxyRemoteCAPath(cr), + SkipCAVerification: skipVerify, + }, + }, + } + switch mux.Role { + case temporalv1alpha1.ProxyRoleServer: + if mux.Server == nil { + return "", fmt.Errorf("mux.server is required for role=server") + } + remote.ConnectionType = "mux-server" + remote.MuxAddressInfo.Address = fmt.Sprintf("0.0.0.0:%d", mux.Server.ListenPort) + case temporalv1alpha1.ProxyRoleClient: + if mux.Client == nil { + return "", fmt.Errorf("mux.client is required for role=client") + } + remote.ConnectionType = "mux-client" + remote.MuxAddressInfo.Address = mux.Client.ServerAddress + // s2s-proxy requires the mux-client to verify the server certificate by + // name unless verification is skipped. Use the explicit override, else + // derive it from the server address host (matches the server proxy cert). + if !skipVerify { + remote.MuxAddressInfo.TLS.CAServerName = proxyClientCAServerName(mux) + } + default: + return "", fmt.Errorf("unknown mux.role %q", mux.Role) + } + + conn := proxyConnection{ + Name: cr.Name, + Local: proxyLocal{ + ConnectionType: "tcp", + TCPClient: proxyTCPEndpoint{Address: localFrontendAddress}, + TCPServer: proxyTCPEndpoint{Address: fmt.Sprintf("0.0.0.0:%d", ProxyTCPServerPort)}, + }, + Remote: remote, + ACLPolicy: buildACLPolicy(cr), + } + if t := cr.Spec.Translation; t != nil { + if len(t.Namespaces) > 0 { + nt := &proxyNamespaceTranslation{} + for _, m := range t.Namespaces { + nt.Mappings = append(nt.Mappings, proxyNamespaceMapping{Local: m.Local, Remote: m.Remote}) + } + conn.NamespaceTranslation = nt + } + if len(t.SearchAttributes) > 0 { + st := &proxySATranslation{} + for _, sa := range t.SearchAttributes { + m := proxySANamespaceMapping{Name: sa.Namespace, NamespaceID: sa.Namespace} + for _, f := range sa.Mappings { + m.Mappings = append(m.Mappings, proxyFieldMapping{LocalFieldName: f.LocalFieldName, RemoteFieldName: f.RemoteFieldName}) + } + st.NamespaceMappings = append(st.NamespaceMappings, m) + } + conn.SearchAttributeTranslation = st + } + } + if f := cr.Spec.FailoverVersionIncrement; f != nil { + conn.FailoverVersionIncrementTranslation = &proxyFailover{Local: f.Local, Remote: f.Remote} + } + + file := proxyConfigFile{ClusterConnections: []proxyConnection{conn}} + raw, err := sigsyaml.Marshal(file) + if err != nil { + return "", fmt.Errorf("marshal proxy config: %w", err) + } + return string(raw), nil +} + +func proxyRemoteCAPath(cr *temporalv1alpha1.TemporalClusterProxy) string { + if cr.Spec.Mux.TLS.PeerCARef != nil { + return ProxyPeerCAMountPath + "/ca.crt" + } + return ProxyTLSMountPath + "/ca.crt" +} + +// proxyClientCAServerName returns the TLS server name a mux-client verifies the +// remote server certificate against. It uses the explicit CAServerName override +// when set, otherwise the host portion of the client's server address. +func proxyClientCAServerName(mux temporalv1alpha1.ProxyMux) string { + if mux.TLS.CAServerName != "" { + return mux.TLS.CAServerName + } + if mux.Client == nil { + return "" + } + if host, _, err := net.SplitHostPort(mux.Client.ServerAddress); err == nil { + return host + } + return mux.Client.ServerAddress +} + +func buildACLPolicy(cr *temporalv1alpha1.TemporalClusterProxy) *proxyACLPolicy { + methods := DefaultAllowedAdminMethods() + var namespaces []string + if a := cr.Spec.ACL; a != nil { + if len(a.AllowedAdminMethods) > 0 { + methods = a.AllowedAdminMethods + } + namespaces = a.AllowedNamespaces + } + return &proxyACLPolicy{ + AllowedMethods: map[string][]string{"adminService": methods}, + AllowedNamespaces: namespaces, + } +} diff --git a/internal/webhook/v1alpha1/temporalcluster_webhook.go b/internal/webhook/v1alpha1/temporalcluster_webhook.go index f4a4514..7c28a4d 100644 --- a/internal/webhook/v1alpha1/temporalcluster_webhook.go +++ b/internal/webhook/v1alpha1/temporalcluster_webhook.go @@ -222,7 +222,7 @@ func validatePersistence(cluster *temporalv1alpha1.TemporalCluster, specPath *fi } func validateMTLS(cluster *temporalv1alpha1.TemporalCluster, specPath *field.Path) field.ErrorList { - if cluster.Spec.MTLS != nil && cluster.Spec.MTLS.Provider == "cert-manager" && cluster.Spec.MTLS.IssuerRef == nil { + if cluster.Spec.MTLS != nil && cluster.Spec.MTLS.Provider == tlsProviderCertManager && cluster.Spec.MTLS.IssuerRef == nil { return field.ErrorList{field.Required(specPath.Child("mtls", "issuerRef"), "issuerRef is required when mtls.provider is cert-manager")} } diff --git a/internal/webhook/v1alpha1/temporalclusterproxy_webhook.go b/internal/webhook/v1alpha1/temporalclusterproxy_webhook.go new file mode 100644 index 0000000..f958869 --- /dev/null +++ b/internal/webhook/v1alpha1/temporalclusterproxy_webhook.go @@ -0,0 +1,168 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +// defaultProxyImage is the pinned s2s-proxy image used when Spec.Image is unset. +const defaultProxyImage = "temporalio/s2s-proxy:v0.2.1" + +const ( + tlsProviderCertManager = "cert-manager" + tlsProviderSecret = "secret" +) + +var temporalclusterproxylog = logf.Log.WithName("temporalclusterproxy-resource") + +// SetupTemporalClusterProxyWebhookWithManager registers the webhook for +// TemporalClusterProxy in the manager. +func SetupTemporalClusterProxyWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr, &temporalv1alpha1.TemporalClusterProxy{}). + WithValidator(&TemporalClusterProxyCustomValidator{}). + WithDefaulter(&TemporalClusterProxyCustomDefaulter{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-temporal-bmor10-com-v1alpha1-temporalclusterproxy,mutating=false,failurePolicy=fail,sideEffects=None,groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=create;update,versions=v1alpha1,name=vtemporalclusterproxy-v1alpha1.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-temporal-bmor10-com-v1alpha1-temporalclusterproxy,mutating=true,failurePolicy=fail,sideEffects=None,groups=temporal.bmor10.com,resources=temporalclusterproxies,verbs=create;update,versions=v1alpha1,name=mtemporalclusterproxy-v1alpha1.kb.io,admissionReviewVersions=v1 + +// TemporalClusterProxyCustomDefaulter defaults optional fields on TemporalClusterProxy. +type TemporalClusterProxyCustomDefaulter struct{} + +var _ admission.Defaulter[*temporalv1alpha1.TemporalClusterProxy] = &TemporalClusterProxyCustomDefaulter{} + +func (d *TemporalClusterProxyCustomDefaulter) Default(_ context.Context, cr *temporalv1alpha1.TemporalClusterProxy) error { + if cr.Spec.Mux.TLS.Provider == "" { + cr.Spec.Mux.TLS.Provider = tlsProviderCertManager + } + if cr.Spec.Peer.EnableConnection == nil { + enable := true + cr.Spec.Peer.EnableConnection = &enable + } + if cr.Spec.Image == "" { + cr.Spec.Image = defaultProxyImage + } + return nil +} + +// TemporalClusterProxyCustomValidator validates TemporalClusterProxy resources. +type TemporalClusterProxyCustomValidator struct{} + +var _ admission.Validator[*temporalv1alpha1.TemporalClusterProxy] = &TemporalClusterProxyCustomValidator{} + +func validateMuxRole(cr *temporalv1alpha1.TemporalClusterProxy) field.ErrorList { + var errs field.ErrorList + muxPath := field.NewPath("spec", "mux") + switch cr.Spec.Mux.Role { + case temporalv1alpha1.ProxyRoleServer: + if cr.Spec.Mux.Server == nil { + errs = append(errs, field.Required(muxPath.Child("server"), "server is required for role=server")) + } + if cr.Spec.Mux.Client != nil { + errs = append(errs, field.Invalid(muxPath.Child("client"), "client", "client must be unset for role=server")) + } + case temporalv1alpha1.ProxyRoleClient: + if cr.Spec.Mux.Client == nil || cr.Spec.Mux.Client.ServerAddress == "" { + errs = append(errs, field.Required(muxPath.Child("client", "serverAddress"), "client.serverAddress is required for role=client")) + } + if cr.Spec.Mux.Server != nil { + errs = append(errs, field.Invalid(muxPath.Child("server"), "server", "server must be unset for role=client")) + } + default: + errs = append(errs, field.NotSupported(muxPath.Child("role"), cr.Spec.Mux.Role, + []string{temporalv1alpha1.ProxyRoleServer, temporalv1alpha1.ProxyRoleClient})) + } + return errs +} + +func validateMuxTLS(cr *temporalv1alpha1.TemporalClusterProxy) field.ErrorList { + var errs field.ErrorList + tlsPath := field.NewPath("spec", "mux", "tls") + switch cr.Spec.Mux.TLS.Provider { + case "", tlsProviderCertManager: + if cr.Spec.Mux.TLS.IssuerRef == nil || cr.Spec.Mux.TLS.IssuerRef.Name == "" { + errs = append(errs, field.Required(tlsPath.Child("issuerRef"), "issuerRef is required for provider=cert-manager")) + } + case tlsProviderSecret: + if cr.Spec.Mux.TLS.SecretRef == nil || cr.Spec.Mux.TLS.SecretRef.Name == "" { + errs = append(errs, field.Required(tlsPath.Child("secretRef"), "secretRef is required for provider=secret")) + } + default: + errs = append(errs, field.NotSupported(tlsPath.Child("provider"), cr.Spec.Mux.TLS.Provider, + []string{tlsProviderCertManager, tlsProviderSecret})) + } + return errs +} + +func validateClusterProxy(cr *temporalv1alpha1.TemporalClusterProxy) field.ErrorList { + var errs field.ErrorList + specPath := field.NewPath("spec") + + if cr.Spec.LocalClusterRef.Name == "" { + errs = append(errs, field.Required(specPath.Child("localClusterRef", "name"), "local cluster reference is required")) + } + if cr.Spec.Peer.Name == "" { + errs = append(errs, field.Required(specPath.Child("peer", "name"), "peer name is required")) + } + if cr.Spec.Peer.Name != "" && cr.Spec.Peer.Name == cr.Spec.LocalClusterName { + errs = append(errs, field.Invalid(specPath.Child("peer", "name"), cr.Spec.Peer.Name, "peer name must differ from localClusterName")) + } + + errs = append(errs, validateMuxRole(cr)...) + errs = append(errs, validateMuxTLS(cr)...) + return errs +} + +// ValidateCreate implements admission.Validator. +func (v *TemporalClusterProxyCustomValidator) ValidateCreate(_ context.Context, cr *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + temporalclusterproxylog.Info("validate create", "name", cr.GetName()) + if errs := validateClusterProxy(cr); len(errs) > 0 { + return nil, errs.ToAggregate() + } + return nil, nil +} + +// ValidateUpdate implements admission.Validator. +func (v *TemporalClusterProxyCustomValidator) ValidateUpdate(_ context.Context, oldCR, newCR *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + temporalclusterproxylog.Info("validate update", "name", newCR.GetName()) + var errs field.ErrorList + if oldCR.Spec.Mux.Role != newCR.Spec.Mux.Role { + errs = append(errs, field.Forbidden(field.NewPath("spec", "mux", "role"), "mux.role is immutable")) + } + if oldCR.Spec.LocalClusterRef.Name != newCR.Spec.LocalClusterRef.Name { + errs = append(errs, field.Forbidden(field.NewPath("spec", "localClusterRef"), "localClusterRef is immutable")) + } + errs = append(errs, validateClusterProxy(newCR)...) + if len(errs) > 0 { + return nil, errs.ToAggregate() + } + return nil, nil +} + +// ValidateDelete implements admission.Validator. +func (v *TemporalClusterProxyCustomValidator) ValidateDelete(_ context.Context, _ *temporalv1alpha1.TemporalClusterProxy) (admission.Warnings, error) { + return nil, nil +} diff --git a/internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go b/internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go new file mode 100644 index 0000000..b40d25f --- /dev/null +++ b/internal/webhook/v1alpha1/temporalclusterproxy_webhook_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2026 Brian Morton. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "testing" + + temporalv1alpha1 "github.com/bmorton/temporal-operator/api/v1alpha1" +) + +func validServerProxy() *temporalv1alpha1.TemporalClusterProxy { + return &temporalv1alpha1.TemporalClusterProxy{ + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + LocalClusterRef: temporalv1alpha1.ClusterReference{Name: "local-cluster"}, + LocalClusterName: "local", + Peer: temporalv1alpha1.ProxyPeer{ + Name: "remote", + }, + Mux: temporalv1alpha1.ProxyMux{ + Role: temporalv1alpha1.ProxyRoleServer, + Server: &temporalv1alpha1.ProxyMuxServer{ + ListenPort: 7600, + }, + TLS: temporalv1alpha1.ProxyMuxTLS{ + Provider: "cert-manager", + IssuerRef: &temporalv1alpha1.IssuerReference{Name: "my-issuer"}, + }, + }, + }, + } +} + +func TestTemporalClusterProxyValidateCreate(t *testing.T) { + v := &TemporalClusterProxyCustomValidator{} + ctx := context.Background() + + cases := []struct { + name string + obj *temporalv1alpha1.TemporalClusterProxy + wantErr bool + }{ + { + name: "valid server proxy", + obj: validServerProxy(), + wantErr: false, + }, + { + name: "server role without server block", + obj: func() *temporalv1alpha1.TemporalClusterProxy { + p := validServerProxy() + p.Spec.Mux.Server = nil + return p + }(), + wantErr: true, + }, + { + name: "client role without client block", + obj: func() *temporalv1alpha1.TemporalClusterProxy { + p := validServerProxy() + p.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + p.Spec.Mux.Server = nil + return p + }(), + wantErr: true, + }, + { + name: "cert-manager provider without issuerRef", + obj: func() *temporalv1alpha1.TemporalClusterProxy { + p := validServerProxy() + p.Spec.Mux.TLS.IssuerRef = nil + return p + }(), + wantErr: true, + }, + { + name: "secret provider without secretRef", + obj: func() *temporalv1alpha1.TemporalClusterProxy { + p := validServerProxy() + p.Spec.Mux.TLS.Provider = "secret" + p.Spec.Mux.TLS.IssuerRef = nil + return p + }(), + wantErr: true, + }, + { + name: "peer name equals localClusterName", + obj: func() *temporalv1alpha1.TemporalClusterProxy { + p := validServerProxy() + p.Spec.Peer.Name = "local" + return p + }(), + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := v.ValidateCreate(ctx, tc.obj) + if tc.wantErr != (err != nil) { + t.Fatalf("wantErr=%v got err=%v", tc.wantErr, err) + } + }) + } +} + +func TestTemporalClusterProxyValidateUpdate(t *testing.T) { + v := &TemporalClusterProxyCustomValidator{} + ctx := context.Background() + + t.Run("immutable mux.role", func(t *testing.T) { + old := validServerProxy() + newP := validServerProxy() + newP.Spec.Mux.Role = temporalv1alpha1.ProxyRoleClient + newP.Spec.Mux.Server = nil + newP.Spec.Mux.Client = &temporalv1alpha1.ProxyMuxClient{ServerAddress: "remote:7600"} + _, err := v.ValidateUpdate(ctx, old, newP) + if err == nil { + t.Fatal("expected error for immutable mux.role, got nil") + } + }) + + t.Run("immutable localClusterRef", func(t *testing.T) { + old := validServerProxy() + newP := validServerProxy() + newP.Spec.LocalClusterRef.Name = "different-cluster" + _, err := v.ValidateUpdate(ctx, old, newP) + if err == nil { + t.Fatal("expected error for immutable localClusterRef, got nil") + } + }) + + t.Run("valid update", func(t *testing.T) { + old := validServerProxy() + newP := validServerProxy() + newP.Spec.Peer.Name = "new-remote" + _, err := v.ValidateUpdate(ctx, old, newP) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) +} + +func TestTemporalClusterProxyDefaulter(t *testing.T) { + d := &TemporalClusterProxyCustomDefaulter{} + ctx := context.Background() + + t.Run("defaults are filled", func(t *testing.T) { + p := &temporalv1alpha1.TemporalClusterProxy{ + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + Peer: temporalv1alpha1.ProxyPeer{}, + Mux: temporalv1alpha1.ProxyMux{}, + }, + } + if err := d.Default(ctx, p); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if p.Spec.Mux.TLS.Provider != "cert-manager" { + t.Errorf("expected provider=cert-manager, got %q", p.Spec.Mux.TLS.Provider) + } + if p.Spec.Peer.EnableConnection == nil || !*p.Spec.Peer.EnableConnection { + t.Error("expected peer.enableConnection=true") + } + if p.Spec.Image != defaultProxyImage { + t.Errorf("expected image=%q, got %q", defaultProxyImage, p.Spec.Image) + } + }) + + t.Run("existing values are not overwritten", func(t *testing.T) { + enable := false + p := &temporalv1alpha1.TemporalClusterProxy{ + Spec: temporalv1alpha1.TemporalClusterProxySpec{ + Image: "my-custom-image:v1", + Peer: temporalv1alpha1.ProxyPeer{ + EnableConnection: &enable, + }, + Mux: temporalv1alpha1.ProxyMux{ + TLS: temporalv1alpha1.ProxyMuxTLS{Provider: "secret"}, + }, + }, + } + if err := d.Default(ctx, p); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if p.Spec.Mux.TLS.Provider != "secret" { + t.Errorf("expected provider=secret, got %q", p.Spec.Mux.TLS.Provider) + } + if p.Spec.Peer.EnableConnection == nil || *p.Spec.Peer.EnableConnection { + t.Error("expected peer.enableConnection=false (not overwritten)") + } + if p.Spec.Image != "my-custom-image:v1" { + t.Errorf("expected image=my-custom-image:v1, got %q", p.Spec.Image) + } + }) +} diff --git a/internal/webhook/v1alpha1/webhook_suite_test.go b/internal/webhook/v1alpha1/webhook_suite_test.go index 6e6ea80..4abb704 100644 --- a/internal/webhook/v1alpha1/webhook_suite_test.go +++ b/internal/webhook/v1alpha1/webhook_suite_test.go @@ -124,6 +124,9 @@ var _ = BeforeSuite(func() { err = SetupTemporalClusterConnectionWebhookWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + err = SetupTemporalClusterProxyWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + // +kubebuilder:scaffold:webhook go func() { diff --git a/test/e2e/clusterproxy/00-databases.yaml b/test/e2e/clusterproxy/00-databases.yaml new file mode 100644 index 0000000..4460c75 --- /dev/null +++ b/test/e2e/clusterproxy/00-databases.yaml @@ -0,0 +1,33 @@ +# Creates the four databases used by the two TemporalClusters. CNPG's initdb only +# creates the bootstrap "temporal" database, so this one-shot Job connects with +# the application user (which was granted CREATEDB) and creates the rest if they +# do not already exist. Mirrors ../postgres/02-secrets.yaml but for two clusters. +apiVersion: batch/v1 +kind: Job +metadata: + name: create-databases +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: createdbs + image: postgres:16 + command: ["/bin/sh", "-c"] + # NOTE: use a literal block (|-) and keep each psql invocation on a + # single line. A folded scalar (>-) with more-indented continuation + # lines preserves their newlines, which splits "psql -tc" from its + # query string and breaks the command. + args: + - |- + until pg_isready -h temporal-pg-rw -U temporal; do sleep 2; done + for db in temporal_a temporal_a_visibility temporal_b temporal_b_visibility; do + psql -h temporal-pg-rw -U temporal -d temporal -tc "SELECT 1 FROM pg_database WHERE datname = '$db'" | grep -q 1 || psql -h temporal-pg-rw -U temporal -d temporal -c "CREATE DATABASE $db OWNER temporal;" + done + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: temporal-pg-app + key: password diff --git a/test/e2e/clusterproxy/00-postgres.yaml b/test/e2e/clusterproxy/00-postgres.yaml new file mode 100644 index 0000000..583b6b4 --- /dev/null +++ b/test/e2e/clusterproxy/00-postgres.yaml @@ -0,0 +1,31 @@ +# A single CloudNativePG (CNPG) Postgres instance that backs BOTH Temporal +# clusters in this multi-cluster suite. Each TemporalCluster gets its own pair of +# databases so the two clusters are independent persistence-wise (only Temporal's +# own cross-cluster replication moves data between them). +# +# cluster-a default store : temporal_a +# cluster-a visibility : temporal_a_visibility +# cluster-b default store : temporal_b +# cluster-b visibility : temporal_b_visibility +# +# Prerequisite: the CNPG operator must be installed in the cluster (the e2e +# workflow installs it). initdb creates the "temporal" login role with CREATEDB +# (granted via postInitSQL) and a generated "temporal-pg-app" Secret holding its +# password, referenced by both TemporalClusters. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: temporal-pg +spec: + instances: 1 + storage: + size: 1Gi + postgresql: + parameters: + max_connections: "300" + bootstrap: + initdb: + database: temporal + owner: temporal + postInitSQL: + - ALTER ROLE temporal WITH CREATEDB; diff --git a/test/e2e/clusterproxy/01-assert.yaml b/test/e2e/clusterproxy/01-assert.yaml new file mode 100644 index 0000000..7f54e4c --- /dev/null +++ b/test/e2e/clusterproxy/01-assert.yaml @@ -0,0 +1,31 @@ +# Both clusters must reconcile to Ready=True with their frontends available +# before replication can be wired up. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalCluster +metadata: + name: temporal-cluster-a +status: + (conditions[?type == 'Ready'].status | [0]): "True" + phase: Ready +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: temporal-cluster-a-frontend +status: + (readyReplicas > `0`): true +--- +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalCluster +metadata: + name: temporal-cluster-b +status: + (conditions[?type == 'Ready'].status | [0]): "True" + phase: Ready +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: temporal-cluster-b-frontend +status: + (readyReplicas > `0`): true diff --git a/test/e2e/clusterproxy/01-cluster-a.yaml b/test/e2e/clusterproxy/01-cluster-a.yaml new file mode 100644 index 0000000..5e73e62 --- /dev/null +++ b/test/e2e/clusterproxy/01-cluster-a.yaml @@ -0,0 +1,46 @@ +# Cluster A: the initial ACTIVE cluster of the replication pair. Global namespaces +# are enabled and a distinct currentClusterName/initialFailoverVersion is set. The +# failoverVersionIncrement MUST match cluster B so failover versions are +# coordinated; masterClusterName equals this cluster's own currentClusterName +# (the operator renders only the local cluster into clusterInformation, and +# Temporal requires masterClusterName to be present there). Runs plaintext (no +# mTLS) so the in-cluster admin-tools proof pods can reach the frontend without +# client certificates -- replication between the two frontends also flows +# plaintext inside the kind network. +# +# Host is namespace-qualified via chainsaw's $namespace binding because the +# operator dials Postgres from the temporal-system namespace. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalCluster +metadata: + name: temporal-cluster-a +spec: + version: "1.31.1" + numHistoryShards: 512 + clusterMetadata: + enableGlobalNamespace: true + failoverVersionIncrement: 100 + currentClusterName: "clusterA" + initialFailoverVersion: 1 + masterClusterName: "clusterA" + persistence: + defaultStore: + sql: + pluginName: postgres12 + host: (join('.', ['temporal-pg-rw', $namespace, 'svc.cluster.local'])) + port: 5432 + database: temporal_a + user: temporal + passwordSecretRef: + name: temporal-pg-app + key: password + visibilityStore: + sql: + pluginName: postgres12 + host: (join('.', ['temporal-pg-rw', $namespace, 'svc.cluster.local'])) + port: 5432 + database: temporal_a_visibility + user: temporal + passwordSecretRef: + name: temporal-pg-app + key: password diff --git a/test/e2e/clusterproxy/01-cluster-b.yaml b/test/e2e/clusterproxy/01-cluster-b.yaml new file mode 100644 index 0000000..905af95 --- /dev/null +++ b/test/e2e/clusterproxy/01-cluster-b.yaml @@ -0,0 +1,41 @@ +# Cluster B: the initial STANDBY cluster. Shares failoverVersionIncrement with +# cluster A but has a distinct currentClusterName and initialFailoverVersion. +# masterClusterName must equal this cluster's own currentClusterName: the +# operator renders only the local cluster into clusterInformation (remote peers +# are added dynamically via the connection controller), and Temporal requires +# masterClusterName to be present in clusterInformation. After the failover step +# this cluster becomes the active cluster. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalCluster +metadata: + name: temporal-cluster-b +spec: + version: "1.31.1" + numHistoryShards: 512 + clusterMetadata: + enableGlobalNamespace: true + failoverVersionIncrement: 100 + currentClusterName: "clusterB" + initialFailoverVersion: 2 + masterClusterName: "clusterB" + persistence: + defaultStore: + sql: + pluginName: postgres12 + host: (join('.', ['temporal-pg-rw', $namespace, 'svc.cluster.local'])) + port: 5432 + database: temporal_b + user: temporal + passwordSecretRef: + name: temporal-pg-app + key: password + visibilityStore: + sql: + pluginName: postgres12 + host: (join('.', ['temporal-pg-rw', $namespace, 'svc.cluster.local'])) + port: 5432 + database: temporal_b_visibility + user: temporal + passwordSecretRef: + name: temporal-pg-app + key: password diff --git a/test/e2e/clusterproxy/02-assert.yaml b/test/e2e/clusterproxy/02-assert.yaml new file mode 100644 index 0000000..ffe101e --- /dev/null +++ b/test/e2e/clusterproxy/02-assert.yaml @@ -0,0 +1,7 @@ +# The CA-backed Issuer must be Ready before the proxy Certificates can be minted. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: mux-ca-issuer +status: + (conditions[?type == 'Ready'].status | [0]): "True" diff --git a/test/e2e/clusterproxy/02-issuer.yaml b/test/e2e/clusterproxy/02-issuer.yaml new file mode 100644 index 0000000..bc2452e --- /dev/null +++ b/test/e2e/clusterproxy/02-issuer.yaml @@ -0,0 +1,38 @@ +# cert-manager issuer chain for the mux mTLS certificates. +# +# Both proxy CRs reference the mux-ca-issuer. Because both use the same CA the +# TLS handshake succeeds without a separate peerCARef: each proxy trusts the CA +# bundle embedded in its own cert-manager Certificate secret (which includes the +# CA cert), so client auth and server auth are validated by the shared CA. +# +# Step 1 — SelfSigned issuer: used only to mint the CA certificate itself. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: mux-selfsigned +spec: + selfSigned: {} +--- +# Step 2 — CA certificate: a locally-trusted CA whose secret is referenced by the +# CA issuer below. isCA: true ensures cert-manager embeds it as a CA in the chain. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: mux-ca +spec: + isCA: true + commonName: mux-ca + secretName: mux-ca-secret + duration: 87600h # 10 years + issuerRef: + name: mux-selfsigned + kind: Issuer +--- +# Step 3 — CA-backed issuer: referenced by both TemporalClusterProxy CRs. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: mux-ca-issuer +spec: + ca: + secretName: mux-ca-secret diff --git a/test/e2e/clusterproxy/03-assert.yaml b/test/e2e/clusterproxy/03-assert.yaml new file mode 100644 index 0000000..99d1b11 --- /dev/null +++ b/test/e2e/clusterproxy/03-assert.yaml @@ -0,0 +1,23 @@ +# Both TemporalClusterProxy CRs must reach Ready=True. +# +# Ready=True is only set by the controller when: +# 1. The s2s-proxy Deployment has at least one available replica. +# 2. The local TemporalCluster frontend is ready. +# 3. UpsertRemoteCluster succeeded on the local cluster (peer registered). +# +# Reaching Ready=True on both sides proves: the proxy Deployments came up, the +# cert-manager Certificates were issued, and each cluster's frontend accepted the +# remote-cluster registration via the proxy tcpServer address. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: proxy-server +status: + (conditions[?type == 'Ready'].status | [0]): "True" +--- +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: proxy-client +status: + (conditions[?type == 'Ready'].status | [0]): "True" diff --git a/test/e2e/clusterproxy/03-proxies.yaml b/test/e2e/clusterproxy/03-proxies.yaml new file mode 100644 index 0000000..ee413da --- /dev/null +++ b/test/e2e/clusterproxy/03-proxies.yaml @@ -0,0 +1,55 @@ +# Server proxy: deployed next to cluster-b, opens the mux port (6334) for +# cluster-a's client proxy to dial. ClusterIP is used here because both proxies +# run in the same kind cluster; no LoadBalancer is needed for in-cluster access. +# The operator creates a Service named proxy-server-s2s-proxy which the client +# proxy below targets via in-cluster DNS. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: proxy-server +spec: + localClusterRef: + name: temporal-cluster-b + # Must match temporal-cluster-b's clusterMetadata.currentClusterName. + localClusterName: clusterB + peer: + name: clusterA + mux: + role: server + server: + listenPort: 6334 + # ClusterIP suffices: client proxy reaches this via the in-cluster Service DNS. + exposure: + type: ClusterIP + tls: + provider: cert-manager + issuerRef: + name: mux-ca-issuer +--- +# Client proxy: deployed next to cluster-a, dials the server proxy over the +# in-cluster Service DNS. The serverAddress uses a chainsaw JMESPath expression +# to embed the dynamic test namespace so the address resolves correctly regardless +# of which ephemeral namespace chainsaw creates. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalClusterProxy +metadata: + name: proxy-client +spec: + localClusterRef: + name: temporal-cluster-a + # Must match temporal-cluster-a's clusterMetadata.currentClusterName. + localClusterName: clusterA + peer: + name: clusterB + mux: + role: client + client: + # proxy-server-s2s-proxy is the Service created for the proxy-server CR + # (name = -s2s-proxy). Port 6334 is the mux listen port. + serverAddress: (join(':', [join('.', ['proxy-server-s2s-proxy', $namespace, 'svc.cluster.local']), '6334'])) + tls: + provider: cert-manager + issuerRef: + name: mux-ca-issuer + # No peerCARef needed: both proxies share the same mux-ca-issuer CA, so + # the CA bundle from each proxy's own cert secret is sufficient for mutual auth. diff --git a/test/e2e/clusterproxy/04-assert.yaml b/test/e2e/clusterproxy/04-assert.yaml new file mode 100644 index 0000000..b773b09 --- /dev/null +++ b/test/e2e/clusterproxy/04-assert.yaml @@ -0,0 +1,8 @@ +# The namespace is registered and ready. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalNamespace +metadata: + name: orders +status: + registered: true + (conditions[?type == 'Ready'].status | [0]): "True" diff --git a/test/e2e/clusterproxy/04-namespace.yaml b/test/e2e/clusterproxy/04-namespace.yaml new file mode 100644 index 0000000..410473c --- /dev/null +++ b/test/e2e/clusterproxy/04-namespace.yaml @@ -0,0 +1,16 @@ +# A GLOBAL namespace created against cluster-a as the active cluster. Because +# the proxy mux link is live, registering on cluster-a causes Temporal to +# propagate the namespace to cluster-b through the mux tunnel. +apiVersion: temporal.bmor10.com/v1alpha1 +kind: TemporalNamespace +metadata: + name: orders +spec: + clusterRef: + name: temporal-cluster-a + retentionPeriod: "24h" + description: "Global orders namespace for clusterproxy e2e" + ownerEmail: "team@example.com" + isGlobal: true + clusters: [clusterA, clusterB] + activeCluster: clusterA diff --git a/test/e2e/clusterproxy/chainsaw-test.yaml b/test/e2e/clusterproxy/chainsaw-test.yaml new file mode 100644 index 0000000..cc436e1 --- /dev/null +++ b/test/e2e/clusterproxy/chainsaw-test.yaml @@ -0,0 +1,235 @@ +# Chainsaw test: PROVE that TemporalClusterProxy establishes a working mux +# replication link between two Temporal clusters in a single kind cluster. +# +# Architecture (single-cluster variant): +# - Both TemporalClusters (clusterA active, clusterB standby) share one Postgres +# instance with per-cluster databases, identical to the multicluster suite. +# - A server proxy (proxy-server, next to cluster-b) opens mux port 6334 via +# ClusterIP. A client proxy (proxy-client, next to cluster-a) dials the server +# proxy over in-cluster DNS: proxy-server-s2s-proxy..svc.cluster.local:6334. +# - Both proxies use a shared cert-manager CA (mux-ca-issuer) so mTLS succeeds +# without a separate peerCARef. +# +# Assertions: +# 1. Both TemporalClusters reach Ready=True (same proof as multicluster suite). +# 2. cert-manager CA issuer is Ready. +# 3. Both TemporalClusterProxy CRs reach Ready=True -- proves: +# a. s2s-proxy Deployments came up and pulled the image. +# b. cert-manager issued mux TLS certificates. +# c. Each cluster's frontend accepted UpsertRemoteCluster via the proxy. +# 4. A global namespace created on cluster-a is describable on cluster-b as +# global with active=clusterA -- proves the mux tunnel carries replication +# traffic end-to-end (namespace metadata propagated through the proxy). +# +# This suite passes against a real 2-cluster environment via the nsc runner +# (`SUITE=clusterproxy make chainsaw-test-nsc`); kind-based CI runs it too. It +# faithfully follows the test/e2e/multicluster/ conventions. +# +# Prerequisites (installed by the e2e workflow): +# - temporal-operator + CRDs +# - cert-manager +# - CloudNativePG (CNPG) +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: clusterproxy +spec: + timeouts: + apply: 2m + assert: 15m + exec: 15m + steps: + # ---------------------------------------------------------------------- + # Provision a single Postgres instance with one database pair per cluster. + # Identical to the multicluster suite setup. + # ---------------------------------------------------------------------- + - name: provision-postgres + try: + - apply: + file: 00-postgres.yaml + - assert: + resource: + apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + metadata: + name: temporal-pg + status: + readyInstances: 1 + - apply: + file: 00-databases.yaml + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: create-databases + status: + succeeded: 1 + + # ---------------------------------------------------------------------- + # Bring up both clusters and wait for Ready. + # ---------------------------------------------------------------------- + - name: clusters + try: + - apply: + file: 01-cluster-a.yaml + - apply: + file: 01-cluster-b.yaml + - assert: + file: 01-assert.yaml + catch: + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalCluster + - events: {} + + # ---------------------------------------------------------------------- + # Set up the cert-manager CA issuer used by both proxy TLS certificates. + # A SelfSigned issuer mints the CA cert; a CA-backed issuer derives from it. + # Both proxies reference mux-ca-issuer so they share the same CA root. + # ---------------------------------------------------------------------- + - name: issuer + try: + - apply: + file: 02-issuer.yaml + - assert: + file: 02-assert.yaml + catch: + - describe: + apiVersion: cert-manager.io/v1 + kind: Issuer + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + - events: {} + + # ---------------------------------------------------------------------- + # Deploy the server proxy (next to cluster-b) and the client proxy (next to + # cluster-a). The client proxy's serverAddress uses a JMESPath expression to + # embed the dynamic namespace so the in-cluster DNS resolves correctly. + # + # Wait for both proxies to reach Ready=True. This condition is set only when: + # - the s2s-proxy Deployment has an available replica + # - the local cluster frontend is accepting gRPC + # - UpsertRemoteCluster succeeded (peer registered on the local cluster) + # ---------------------------------------------------------------------- + - name: proxies + try: + - apply: + file: 03-proxies.yaml + - assert: + file: 03-assert.yaml + catch: + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalClusterProxy + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + - events: {} + + # ---------------------------------------------------------------------- + # Create a GLOBAL namespace on cluster-a. Because the mux link is live, + # Temporal propagates namespace metadata to cluster-b through the proxy. + # ---------------------------------------------------------------------- + - name: namespace + try: + - apply: + file: 04-namespace.yaml + - assert: + file: 04-assert.yaml + catch: + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalNamespace + + # ====================================================================== + # PROOF: namespace metadata replicated through the proxy mux. + # Confirm the namespace is global+active=clusterA on cluster-a, then + # assert the SAME namespace is describable on cluster-b (bounded retry + # for async propagation through the mux tunnel). + # ====================================================================== + - name: proof-namespace-replicated-via-proxy + try: + - script: + env: + - name: NS + value: ($namespace) + content: | + set -eu + # Sanity: cluster-a reports the namespace global with active=clusterA. + outa=$(kubectl run -n "$NS" nsdesc-a --rm -i --restart=Never \ + --image=temporalio/admin-tools:1.31.1 -- \ + temporal operator namespace describe --namespace orders \ + --address temporal-cluster-a-frontend:7233 -o json 2>/dev/null || true) + echo "$outa" + packed=$(echo "$outa" | tr -d '[:space:]') + echo "$packed" | grep -q '"isGlobalNamespace":true' \ + || { echo "PROOF FAIL: orders not global on clusterA"; exit 1; } + echo "$packed" | grep -q '"activeClusterName":"clusterA"' \ + || { echo "PROOF FAIL: orders not active=clusterA on clusterA"; exit 1; } + echo "PROOF OK: orders is global active=clusterA on clusterA" + + # Replicated via proxy mux: cluster-b must learn the namespace as + # global active=clusterA within the bounded poll window. + for i in $(seq 1 30); do + outb=$(kubectl run -n "$NS" "nsdesc-b-${i}" --rm -i --restart=Never \ + --image=temporalio/admin-tools:1.31.1 -- \ + temporal operator namespace describe --namespace orders \ + --address temporal-cluster-b-frontend:7233 -o json 2>/dev/null || true) + echo "--- clusterB describe attempt ${i} ---" + echo "$outb" + packedb=$(echo "$outb" | tr -d '[:space:]') + if echo "$packedb" | grep -q '"isGlobalNamespace":true' \ + && echo "$packedb" | grep -q '"activeClusterName":"clusterA"'; then + echo "PROOF OK: orders replicated to clusterB via proxy mux (global, active=clusterA)" + exit 0 + fi + sleep 10 + done + echo "PROOF FAIL: orders never replicated to clusterB via proxy mux" + exit 1 + + # Runs on ANY step failure so operator-managed CRs/pods/events are inspectable. + catch: + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalCluster + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalClusterProxy + - describe: + apiVersion: temporal.bmor10.com/v1alpha1 + kind: TemporalNamespace + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + - script: + env: + - name: NS + value: ($namespace) + content: | + set +e + echo "===== PODS =====" + kubectl -n "$NS" get pods -o wide + echo "===== DEPLOYMENTS =====" + kubectl -n "$NS" get deploy + echo "===== SERVICES =====" + kubectl -n "$NS" get svc + echo "===== EVENTS =====" + kubectl -n "$NS" get events --sort-by=.lastTimestamp | tail -80 + echo "===== PROXY LOGS =====" + for inst in proxy-server proxy-client; do + for p in $(kubectl -n "$NS" get pods -l app.kubernetes.io/instance=$inst -o name 2>/dev/null); do + echo "----- $p -----" + kubectl -n "$NS" logs "$p" --all-containers --tail=80 + done + done + echo "===== CLUSTER LOGS =====" + for inst in temporal-cluster-a temporal-cluster-b; do + for p in $(kubectl -n "$NS" get pods -l app.kubernetes.io/instance=$inst -o name 2>/dev/null); do + echo "----- $p -----" + kubectl -n "$NS" logs "$p" --all-containers --tail=40 + done + done + - events: {}