From 922745ed73c1f52b001d5f1448b82f3f32fcba24 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 03:06:09 +0000 Subject: [PATCH 1/7] Add read-only reader sites --- .github/kind/e2e-calico.yaml | 1 + api/v1alpha1/site_helpers.go | 23 + api/v1alpha1/site_helpers_test.go | 62 ++ api/v1alpha1/types.go | 83 ++- api/v1alpha1/zz_generated.deepcopy.go | 41 +- .../shipstream.io_mysqlfailovergroups.yaml | 103 +++- .../shipstream.io_mysqlstandbyclusters.yaml | 89 ++- .../shipstream.io_mysqlfailovergroups.yaml | 103 +++- .../shipstream.io_mysqlstandbyclusters.yaml | 89 ++- docs/docs/app-integration.mdx | 46 +- docs/docs/backup-overview.mdx | 10 + docs/docs/configuration.mdx | 68 +++ docs/docs/crd-reference.mdx | 86 ++- docs/docs/examples.mdx | 2 +- docs/docs/failover.mdx | 54 +- docs/docs/known-limitations.mdx | 14 +- docs/docs/kubectl-plugin.mdx | 23 +- docs/docs/log-schema.mdx | 21 +- docs/docs/monitoring-prometheus.mdx | 28 + docs/docs/monitoring.mdx | 1 + docs/docs/multi-site.mdx | 111 +++- docs/docs/operations.mdx | 53 +- docs/docs/planned-failover.mdx | 29 +- docs/docs/playground.mdx | 26 +- examples/minimal-failovergroup.yaml | 18 + internal/controller/archiver_poller.go | 3 +- internal/controller/backup_job.go | 3 +- internal/controller/backup_reconciler.go | 39 +- internal/controller/backup_reconciler_test.go | 42 +- internal/controller/credentials.go | 8 +- internal/controller/mysql_tls.go | 8 + internal/controller/mysql_tls_test.go | 3 + internal/controller/reclone_test.go | 8 + internal/controller/reconciler.go | 503 ++++++++++++---- internal/controller/reconciler_test.go | 444 +++++++++++++- internal/controller/runner.go | 101 +++- internal/controller/runner_test.go | 99 +++- internal/controller/source_convergence.go | 272 +++++++++ .../controller/source_convergence_test.go | 136 +++++ internal/controller/topology.go | 522 ++++++++++++----- internal/controller/topology_test.go | 368 ++++++++++-- internal/controller/updater.go | 143 +++++ internal/controller/updater_test.go | 151 +++++ internal/metrics/metrics.go | 10 +- internal/metrics/metrics_test.go | 17 + internal/playground/kube/endpointslices.go | 106 ++++ .../playground/kube/endpointslices_test.go | 95 +++ internal/playground/logs/matcher.go | 37 ++ internal/playground/logs/matcher_test.go | 18 + internal/playground/runner/executor.go | 4 +- internal/playground/runner/profile.go | 4 +- .../runner/profile_registry_test.go | 9 +- internal/playground/scenarios/common.go | 4 +- .../playground/scenarios/inventory_40_test.go | 70 +++ .../scenarios/s02_planned_switchover.go | 68 +++ .../s05_operator_kill_during_failover.go | 4 +- .../s06_self_fence_isolated_primary.go | 27 +- .../s08_gtid_divergence_detection.go | 2 +- .../s10_full_bootstrap_after_data_wipe.go | 4 +- .../scenarios/s11_total_loss_recovery.go | 16 +- .../scenarios/s12_old_primary_recovery.go | 4 +- .../s12_rolling_update_healthy_state.go | 16 +- .../s13_operator_kill_during_bootstrap.go | 4 +- .../scenarios/s16_mysql_process_kill.go | 6 +- .../scenarios/s18_rapid_cr_spec_changes.go | 12 +- .../scenarios/s19_reclone_interlock.go | 2 +- .../s22_replication_status_after_recovery.go | 10 +- .../s34_operator_kill_during_wait_replica.go | 16 +- .../playground/scenarios/s40_observer_test.go | 200 +++++++ .../scenarios/s40_reader_data_loss.go | 541 ++++++++++++++++++ internal/state/matrix.go | 30 +- internal/state/matrix_test.go | 64 +++ playground/chaos-scenarios.md | 51 +- playground/chaos.sh | 36 +- playground/manifests/failovergroup.yaml | 26 +- playground/rebuild.sh | 11 +- playground/setup.sh | 66 ++- test/component/helpers_test.go | 8 + test/component/source_convergence_test.go | 42 ++ test/envtest/reconciler_test.go | 134 ++++- 80 files changed, 5175 insertions(+), 636 deletions(-) create mode 100644 api/v1alpha1/site_helpers_test.go create mode 100644 internal/controller/source_convergence.go create mode 100644 internal/controller/source_convergence_test.go create mode 100644 internal/playground/kube/endpointslices.go create mode 100644 internal/playground/kube/endpointslices_test.go create mode 100644 internal/playground/scenarios/inventory_40_test.go create mode 100644 internal/playground/scenarios/s40_observer_test.go create mode 100644 internal/playground/scenarios/s40_reader_data_loss.go create mode 100644 test/component/source_convergence_test.go diff --git a/.github/kind/e2e-calico.yaml b/.github/kind/e2e-calico.yaml index fd236dc..a798bfe 100644 --- a/.github/kind/e2e-calico.yaml +++ b/.github/kind/e2e-calico.yaml @@ -15,6 +15,7 @@ nodes: - role: control-plane - role: worker - role: worker + - role: worker networking: # Disable kindnet so Calico can manage CNI instead. disableDefaultCNI: true diff --git a/api/v1alpha1/site_helpers.go b/api/v1alpha1/site_helpers.go index a5eab4b..1bef9c0 100644 --- a/api/v1alpha1/site_helpers.go +++ b/api/v1alpha1/site_helpers.go @@ -68,3 +68,26 @@ func (s *MysqlFailoverGroupSpec) PeerSiteNames(name string) []string { } return out } + +// IsReadOnlyReader reports whether the site is a non-promotable serving reader. +func (s SiteSpec) IsReadOnlyReader() bool { + return s.EffectiveRole() == SiteRoleReadOnly +} + +// EffectiveMaxLagSeconds returns the group replication threshold, including +// the API default for objects constructed outside admission. +func (s *MysqlFailoverGroupSpec) EffectiveMaxLagSeconds() int64 { + if s.Replication == nil || s.Replication.MaxLagSeconds == 0 { + return 300 + } + return s.Replication.MaxLagSeconds +} + +// EffectiveReadOnlyMaxLagSeconds returns the reader endpoint threshold. An +// explicit zero is meaningful; nil inherits the group threshold. +func (s *MysqlFailoverGroupSpec) EffectiveReadOnlyMaxLagSeconds() int64 { + if s.Replication != nil && s.Replication.ReadOnlyMaxLagSeconds != nil { + return *s.Replication.ReadOnlyMaxLagSeconds + } + return s.EffectiveMaxLagSeconds() +} diff --git a/api/v1alpha1/site_helpers_test.go b/api/v1alpha1/site_helpers_test.go new file mode 100644 index 0000000..e0e3359 --- /dev/null +++ b/api/v1alpha1/site_helpers_test.go @@ -0,0 +1,62 @@ +package v1alpha1 + +import "testing" + +func TestSiteRoleHelpers(t *testing.T) { + tests := []struct { + name string + role SiteRole + promotable bool + reader bool + }{ + {name: "default", promotable: true}, + {name: "candidate", role: SiteRolePrimaryCandidate, promotable: true}, + {name: "dr only", role: SiteRoleDROnly}, + {name: "reader", role: SiteRoleReadOnly, reader: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := SiteSpec{Role: tt.role} + if got := s.IsPromotable(); got != tt.promotable { + t.Fatalf("IsPromotable() = %v, want %v", got, tt.promotable) + } + if got := s.IsReadOnlyReader(); got != tt.reader { + t.Fatalf("IsReadOnlyReader() = %v, want %v", got, tt.reader) + } + }) + } +} + +func TestPeerSiteNamesIncludesReaders(t *testing.T) { + s := MysqlFailoverGroupSpec{Sites: []SiteSpec{ + {Name: "iad"}, {Name: "pdx"}, {Name: "reader", Role: SiteRoleReadOnly}, + }} + got := s.PeerSiteNames("iad") + if len(got) != 2 || got[0] != "pdx" || got[1] != "reader" { + t.Fatalf("PeerSiteNames() = %v", got) + } +} + +func TestEffectiveReplicationLag(t *testing.T) { + zero := int64(0) + tests := []struct { + name string + spec MysqlFailoverGroupSpec + max int64 + read int64 + }{ + {name: "defaults", max: 300, read: 300}, + {name: "inherits", spec: MysqlFailoverGroupSpec{Replication: &ReplicationSpec{MaxLagSeconds: 42}}, max: 42, read: 42}, + {name: "explicit zero", spec: MysqlFailoverGroupSpec{Replication: &ReplicationSpec{MaxLagSeconds: 42, ReadOnlyMaxLagSeconds: &zero}}, max: 42, read: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.spec.EffectiveMaxLagSeconds(); got != tt.max { + t.Fatalf("EffectiveMaxLagSeconds() = %d, want %d", got, tt.max) + } + if got := tt.spec.EffectiveReadOnlyMaxLagSeconds(); got != tt.read { + t.Fatalf("EffectiveReadOnlyMaxLagSeconds() = %d, want %d", got, tt.read) + } + }) + } +} diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index de125e5..b46d430 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -35,6 +35,10 @@ type MysqlFailoverGroupList struct { // +kubebuilder:validation:XValidation:rule="self.sites.all(x, self.sites.filter(y, y.name == x.name).size() == 1)",message="spec.sites[].name must be unique" // +kubebuilder:validation:XValidation:rule="self.sites.filter(s, s.role == 'primary-candidate').size() >= 2",message="spec.sites must contain at least two sites with role 'primary-candidate'" // +kubebuilder:validation:XValidation:rule="!has(self.splitBrainPolicy) || !has(self.splitBrainPolicy.sitePriorities) || self.splitBrainPolicy.sitePriorities.all(p, self.sites.exists(s, s.name == p && s.role == 'primary-candidate'))",message="splitBrainPolicy.sitePriorities entries must match the names of sites with role 'primary-candidate'" +// +kubebuilder:validation:XValidation:rule="!has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) || (has(self.serviceTemplate.type) && self.serviceTemplate.type in ['NodePort', 'LoadBalancer'])",message="serviceTemplate.externalTrafficPolicy requires serviceTemplate.type NodePort or LoadBalancer" +// +kubebuilder:validation:XValidation:rule="self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.externalTrafficPolicy) || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) && has(self.serviceTemplate.type) ? self.serviceTemplate.type : 'ClusterIP')) in ['NodePort', 'LoadBalancer']))",message="site serviceTemplate.externalTrafficPolicy requires an effective type of NodePort or LoadBalancer" +// +kubebuilder:validation:XValidation:rule="self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.nodePort) || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) && has(self.serviceTemplate.type) ? self.serviceTemplate.type : 'ClusterIP')) in ['NodePort', 'LoadBalancer']))",message="site serviceTemplate.nodePort requires an effective type of NodePort or LoadBalancer" +// +kubebuilder:validation:XValidation:rule="self.sites.all(s, !has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) || ((has(s.serviceTemplate) && has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate.type) ? self.serviceTemplate.type : 'ClusterIP')) in ['NodePort', 'LoadBalancer']))",message="inherited externalTrafficPolicy requires each effective site service type to be NodePort or LoadBalancer" type MysqlFailoverGroupSpec struct { // Image is the MySQL container image. Default: mysql:9.7 // +kubebuilder:default="mysql:9.7" @@ -249,6 +253,13 @@ type ReplicationSpec struct { // +kubebuilder:default=300 // +kubebuilder:validation:Minimum=0 MaxLagSeconds int64 `json:"maxLagSeconds,omitempty"` + + // ReadOnlyMaxLagSeconds is the maximum lag for a read-only site's + // client-facing endpoint. Nil inherits MaxLagSeconds; explicit zero + // requires the reader to be fully caught up. + // +optional + // +kubebuilder:validation:Minimum=0 + ReadOnlyMaxLagSeconds *int64 `json:"readOnlyMaxLagSeconds,omitempty"` } // SiteRole describes the promotion eligibility of a site. @@ -269,7 +280,7 @@ type ReplicationSpec struct { // *promotion* and *policy eligibility*; it does not exempt the site // from the same safety fencing that protects primary-candidate // losers. -// +kubebuilder:validation:Enum=primary-candidate;dr-only +// +kubebuilder:validation:Enum=primary-candidate;dr-only;read-only type SiteRole string const ( @@ -280,9 +291,14 @@ const ( // SiteRoleDROnly is a passive replica role: a site that follows // the active primary but is never promoted automatically. SiteRoleDROnly SiteRole = "dr-only" + + // SiteRoleReadOnly is a serving replica that is never promoted and + // does not participate in core failover-group readiness. + SiteRoleReadOnly SiteRole = "read-only" ) // SiteSpec defines the configuration for a single site in the failover group. +// +kubebuilder:validation:XValidation:rule="self.role == 'read-only' || (has(self.taintNodeSelector) && has(self.lbIP))",message="taintNodeSelector and lbIP are required unless role is read-only" type SiteSpec struct { // Name is the site identifier (e.g. "iad", "pdx", "lhr"). // Must be unique within spec.sites. MaxLength caps the CEL cost @@ -306,17 +322,27 @@ type SiteSpec struct { // group-scoped labels so one physical node can participate in multiple // failover groups at the same site. // +kubebuilder:validation:MinProperties=1 - TaintNodeSelector map[string]string `json:"taintNodeSelector"` + // +optional + TaintNodeSelector map[string]string `json:"taintNodeSelector,omitempty"` // LBIP is the load balancer IP for DNS failover. // +kubebuilder:validation:MinLength=1 - LBIP string `json:"lbIP"` + // +optional + LBIP string `json:"lbIP,omitempty"` // Storage configures the persistent volume for this site. Storage StorageSpec `json:"storage"` // Resources defines the compute resources for the MySQL container. Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // MysqlConf overrides group-level MySQL settings for this site. + MysqlConf map[string]string `json:"mysqlConf,omitempty"` + + // ServiceTemplate overrides the group Service template for this site's + // client-facing MySQL Service. + // +optional + ServiceTemplate *SiteServiceTemplate `json:"serviceTemplate,omitempty"` } // EffectiveRole returns the site's role, defaulting to @@ -415,10 +441,38 @@ type ServiceTemplate struct { // +kubebuilder:validation:Enum=ClusterIP;LoadBalancer;NodePort Type corev1.ServiceType `json:"type,omitempty"` + // ExternalTrafficPolicy controls external traffic routing for Services + // whose effective type is NodePort or LoadBalancer. + // +optional + // +kubebuilder:validation:Enum=Cluster;Local + ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + // Annotations are additional annotations applied to every Service managed by this failover group. Annotations map[string]string `json:"annotations,omitempty"` } +// SiteServiceTemplate customizes one client-facing site Service. +type SiteServiceTemplate struct { + // Type overrides the group Service type. + // +optional + // +kubebuilder:validation:Enum=ClusterIP;LoadBalancer;NodePort + Type corev1.ServiceType `json:"type,omitempty"` + + // ExternalTrafficPolicy overrides the group policy. + // +optional + // +kubebuilder:validation:Enum=Cluster;Local + ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + + // NodePort requests a specific port for this site's MySQL endpoint. + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + NodePort int32 `json:"nodePort,omitempty"` + + // Annotations are merged over group Service annotations for this site. + Annotations map[string]string `json:"annotations,omitempty"` +} + // CredentialsSpec configures per-role MySQL credential management. // Each field references a Secret with 'username' and 'password' keys. // The operator creates MySQL users with role-appropriate privileges @@ -566,8 +620,31 @@ type SiteStatus struct { // DivergentTransactionCount is the number of divergent transactions. // +optional DivergentTransactionCount *int64 `json:"divergentTransactionCount,omitempty"` + + // SourceHost is the replication source reported by MySQL. + // +optional + SourceHost string `json:"sourceHost,omitempty"` + + // SourceConvergenceState reports whether this follower directly follows + // the confirmed active primary. + // +optional + SourceConvergenceState SourceConvergenceState `json:"sourceConvergenceState,omitempty"` + + // SourceConvergenceReason is a stable machine-readable explanation. + // +optional + SourceConvergenceReason string `json:"sourceConvergenceReason,omitempty"` } +// SourceConvergenceState describes direct-primary source convergence. +// +kubebuilder:validation:Enum="";Converged;Pending;Blocked +type SourceConvergenceState string + +const ( + SourceConvergenceConverged SourceConvergenceState = "Converged" + SourceConvergencePending SourceConvergenceState = "Pending" + SourceConvergenceBlocked SourceConvergenceState = "Blocked" +) + func init() { SchemeBuilder.Register(&MysqlFailoverGroup{}, &MysqlFailoverGroupList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dc70014..ae04279 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -994,7 +994,7 @@ func (in *MysqlFailoverGroupSpec) DeepCopyInto(out *MysqlFailoverGroupSpec) { if in.Replication != nil { in, out := &in.Replication, &out.Replication *out = new(ReplicationSpec) - **out = **in + (*in).DeepCopyInto(*out) } in.SidecarResources.DeepCopyInto(&out.SidecarResources) if in.TerminationGracePeriodSeconds != nil { @@ -1509,6 +1509,11 @@ func (in *PointInTimeVerificationSpec) DeepCopy() *PointInTimeVerificationSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicationSpec) DeepCopyInto(out *ReplicationSpec) { *out = *in + if in.ReadOnlyMaxLagSeconds != nil { + in, out := &in.ReadOnlyMaxLagSeconds, &out.ReadOnlyMaxLagSeconds + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationSpec. @@ -1725,6 +1730,28 @@ func (in *SidecarSpec) DeepCopy() *SidecarSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SiteServiceTemplate) DeepCopyInto(out *SiteServiceTemplate) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SiteServiceTemplate. +func (in *SiteServiceTemplate) DeepCopy() *SiteServiceTemplate { + if in == nil { + return nil + } + out := new(SiteServiceTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SiteSpec) DeepCopyInto(out *SiteSpec) { *out = *in @@ -1737,6 +1764,18 @@ func (in *SiteSpec) DeepCopyInto(out *SiteSpec) { } in.Storage.DeepCopyInto(&out.Storage) in.Resources.DeepCopyInto(&out.Resources) + if in.MysqlConf != nil { + in, out := &in.MysqlConf, &out.MysqlConf + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ServiceTemplate != nil { + in, out := &in.ServiceTemplate, &out.ServiceTemplate + *out = new(SiteServiceTemplate) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SiteSpec. diff --git a/charts/bloodraven/crds/shipstream.io_mysqlfailovergroups.yaml b/charts/bloodraven/crds/shipstream.io_mysqlfailovergroups.yaml index b39ce86..566401b 100644 --- a/charts/bloodraven/crds/shipstream.io_mysqlfailovergroups.yaml +++ b/charts/bloodraven/crds/shipstream.io_mysqlfailovergroups.yaml @@ -5708,6 +5708,14 @@ spec: format: int64 minimum: 0 type: integer + readOnlyMaxLagSeconds: + description: |- + ReadOnlyMaxLagSeconds is the maximum lag for a read-only site's + client-facing endpoint. Nil inherits MaxLagSeconds; explicit zero + requires the reader to be fully caught up. + format: int64 + minimum: 0 + type: integer type: object restoreInPlace: description: |- @@ -5942,6 +5950,14 @@ spec: description: Annotations are additional annotations applied to every Service managed by this failover group. type: object + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy controls external traffic routing for Services + whose effective type is NodePort or LoadBalancer. + enum: + - Cluster + - Local + type: string type: description: 'Type is the Kubernetes Service type. Default: ClusterIP' enum: @@ -6068,6 +6084,12 @@ spec: description: LBIP is the load balancer IP for DNS failover. minLength: 1 type: string + mysqlConf: + additionalProperties: + type: string + description: MysqlConf overrides group-level MySQL settings + for this site. + type: object name: description: |- Name is the site identifier (e.g. "iad", "pdx", "lhr"). @@ -6144,7 +6166,40 @@ spec: enum: - primary-candidate - dr-only + - read-only type: string + serviceTemplate: + description: |- + ServiceTemplate overrides the group Service template for this site's + client-facing MySQL Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are merged over group Service annotations + for this site. + type: object + externalTrafficPolicy: + description: ExternalTrafficPolicy overrides the group policy. + enum: + - Cluster + - Local + type: string + nodePort: + description: NodePort requests a specific port for this + site's MySQL endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: + description: Type overrides the group Service type. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object storage: description: Storage configures the persistent volume for this site. @@ -6179,12 +6234,15 @@ spec: minLength: 1 type: string required: - - lbIP - name - storage - - taintNodeSelector - zone type: object + x-kubernetes-validations: + - message: taintNodeSelector and lbIP are required unless role is + read-only + rule: self.role == 'read-only' || (has(self.taintNodeSelector) + && has(self.lbIP)) maxItems: 16 minItems: 2 type: array @@ -6280,6 +6338,29 @@ spec: rule: '!has(self.splitBrainPolicy) || !has(self.splitBrainPolicy.sitePriorities) || self.splitBrainPolicy.sitePriorities.all(p, self.sites.exists(s, s.name == p && s.role == ''primary-candidate''))' + - message: serviceTemplate.externalTrafficPolicy requires serviceTemplate.type + NodePort or LoadBalancer + rule: '!has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || (has(self.serviceTemplate.type) && self.serviceTemplate.type in + [''NodePort'', ''LoadBalancer''])' + - message: site serviceTemplate.externalTrafficPolicy requires an effective + type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) + && has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' + - message: site serviceTemplate.nodePort requires an effective type of + NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.nodePort) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) + && has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' + - message: inherited externalTrafficPolicy requires each effective site + service type to be NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate) && has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' status: description: MysqlFailoverGroupStatus defines the observed state of MysqlFailoverGroup. properties: @@ -7057,6 +7138,24 @@ spec: (populated when replication status is enriched). format: int64 type: integer + sourceConvergenceReason: + description: SourceConvergenceReason is a stable machine-readable + explanation. + type: string + sourceConvergenceState: + description: |- + SourceConvergenceState reports whether this follower directly follows + the confirmed active primary. + enum: + - "" + - Converged + - Pending + - Blocked + type: string + sourceHost: + description: SourceHost is the replication source reported by + MySQL. + type: string state: description: 'State is the current state: writable, read-only, unreachable, or unknown.' diff --git a/charts/bloodraven/crds/shipstream.io_mysqlstandbyclusters.yaml b/charts/bloodraven/crds/shipstream.io_mysqlstandbyclusters.yaml index bd576ff..8e88993 100644 --- a/charts/bloodraven/crds/shipstream.io_mysqlstandbyclusters.yaml +++ b/charts/bloodraven/crds/shipstream.io_mysqlstandbyclusters.yaml @@ -5992,6 +5992,14 @@ spec: format: int64 minimum: 0 type: integer + readOnlyMaxLagSeconds: + description: |- + ReadOnlyMaxLagSeconds is the maximum lag for a read-only site's + client-facing endpoint. Nil inherits MaxLagSeconds; explicit zero + requires the reader to be fully caught up. + format: int64 + minimum: 0 + type: integer type: object restoreInPlace: description: |- @@ -6229,6 +6237,14 @@ spec: description: Annotations are additional annotations applied to every Service managed by this failover group. type: object + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy controls external traffic routing for Services + whose effective type is NodePort or LoadBalancer. + enum: + - Cluster + - Local + type: string type: description: 'Type is the Kubernetes Service type. Default: ClusterIP' @@ -6357,6 +6373,12 @@ spec: description: LBIP is the load balancer IP for DNS failover. minLength: 1 type: string + mysqlConf: + additionalProperties: + type: string + description: MysqlConf overrides group-level MySQL settings + for this site. + type: object name: description: |- Name is the site identifier (e.g. "iad", "pdx", "lhr"). @@ -6434,7 +6456,41 @@ spec: enum: - primary-candidate - dr-only + - read-only type: string + serviceTemplate: + description: |- + ServiceTemplate overrides the group Service template for this site's + client-facing MySQL Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are merged over group Service + annotations for this site. + type: object + externalTrafficPolicy: + description: ExternalTrafficPolicy overrides the + group policy. + enum: + - Cluster + - Local + type: string + nodePort: + description: NodePort requests a specific port for + this site's MySQL endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: + description: Type overrides the group Service type. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object storage: description: Storage configures the persistent volume for this site. @@ -6471,12 +6527,15 @@ spec: minLength: 1 type: string required: - - lbIP - name - storage - - taintNodeSelector - zone type: object + x-kubernetes-validations: + - message: taintNodeSelector and lbIP are required unless + role is read-only + rule: self.role == 'read-only' || (has(self.taintNodeSelector) + && has(self.lbIP)) maxItems: 16 minItems: 2 type: array @@ -6574,6 +6633,32 @@ spec: rule: '!has(self.splitBrainPolicy) || !has(self.splitBrainPolicy.sitePriorities) || self.splitBrainPolicy.sitePriorities.all(p, self.sites.exists(s, s.name == p && s.role == ''primary-candidate''))' + - message: serviceTemplate.externalTrafficPolicy requires serviceTemplate.type + NodePort or LoadBalancer + rule: '!has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || (has(self.serviceTemplate.type) && self.serviceTemplate.type + in [''NodePort'', ''LoadBalancer''])' + - message: site serviceTemplate.externalTrafficPolicy requires + an effective type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate) && has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' + - message: site serviceTemplate.nodePort requires an effective + type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.nodePort) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate) && has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' + - message: inherited externalTrafficPolicy requires each effective + site service type to be NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate) && has(s.serviceTemplate.type) + ? s.serviceTemplate.type : (has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' required: - name - spec diff --git a/config/crd/bases/shipstream.io_mysqlfailovergroups.yaml b/config/crd/bases/shipstream.io_mysqlfailovergroups.yaml index b39ce86..566401b 100644 --- a/config/crd/bases/shipstream.io_mysqlfailovergroups.yaml +++ b/config/crd/bases/shipstream.io_mysqlfailovergroups.yaml @@ -5708,6 +5708,14 @@ spec: format: int64 minimum: 0 type: integer + readOnlyMaxLagSeconds: + description: |- + ReadOnlyMaxLagSeconds is the maximum lag for a read-only site's + client-facing endpoint. Nil inherits MaxLagSeconds; explicit zero + requires the reader to be fully caught up. + format: int64 + minimum: 0 + type: integer type: object restoreInPlace: description: |- @@ -5942,6 +5950,14 @@ spec: description: Annotations are additional annotations applied to every Service managed by this failover group. type: object + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy controls external traffic routing for Services + whose effective type is NodePort or LoadBalancer. + enum: + - Cluster + - Local + type: string type: description: 'Type is the Kubernetes Service type. Default: ClusterIP' enum: @@ -6068,6 +6084,12 @@ spec: description: LBIP is the load balancer IP for DNS failover. minLength: 1 type: string + mysqlConf: + additionalProperties: + type: string + description: MysqlConf overrides group-level MySQL settings + for this site. + type: object name: description: |- Name is the site identifier (e.g. "iad", "pdx", "lhr"). @@ -6144,7 +6166,40 @@ spec: enum: - primary-candidate - dr-only + - read-only type: string + serviceTemplate: + description: |- + ServiceTemplate overrides the group Service template for this site's + client-facing MySQL Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are merged over group Service annotations + for this site. + type: object + externalTrafficPolicy: + description: ExternalTrafficPolicy overrides the group policy. + enum: + - Cluster + - Local + type: string + nodePort: + description: NodePort requests a specific port for this + site's MySQL endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: + description: Type overrides the group Service type. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object storage: description: Storage configures the persistent volume for this site. @@ -6179,12 +6234,15 @@ spec: minLength: 1 type: string required: - - lbIP - name - storage - - taintNodeSelector - zone type: object + x-kubernetes-validations: + - message: taintNodeSelector and lbIP are required unless role is + read-only + rule: self.role == 'read-only' || (has(self.taintNodeSelector) + && has(self.lbIP)) maxItems: 16 minItems: 2 type: array @@ -6280,6 +6338,29 @@ spec: rule: '!has(self.splitBrainPolicy) || !has(self.splitBrainPolicy.sitePriorities) || self.splitBrainPolicy.sitePriorities.all(p, self.sites.exists(s, s.name == p && s.role == ''primary-candidate''))' + - message: serviceTemplate.externalTrafficPolicy requires serviceTemplate.type + NodePort or LoadBalancer + rule: '!has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || (has(self.serviceTemplate.type) && self.serviceTemplate.type in + [''NodePort'', ''LoadBalancer''])' + - message: site serviceTemplate.externalTrafficPolicy requires an effective + type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) + && has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' + - message: site serviceTemplate.nodePort requires an effective type of + NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.nodePort) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type : (has(self.serviceTemplate) + && has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' + - message: inherited externalTrafficPolicy requires each effective site + service type to be NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate) && has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate.type) ? self.serviceTemplate.type : ''ClusterIP'')) + in [''NodePort'', ''LoadBalancer'']))' status: description: MysqlFailoverGroupStatus defines the observed state of MysqlFailoverGroup. properties: @@ -7057,6 +7138,24 @@ spec: (populated when replication status is enriched). format: int64 type: integer + sourceConvergenceReason: + description: SourceConvergenceReason is a stable machine-readable + explanation. + type: string + sourceConvergenceState: + description: |- + SourceConvergenceState reports whether this follower directly follows + the confirmed active primary. + enum: + - "" + - Converged + - Pending + - Blocked + type: string + sourceHost: + description: SourceHost is the replication source reported by + MySQL. + type: string state: description: 'State is the current state: writable, read-only, unreachable, or unknown.' diff --git a/config/crd/bases/shipstream.io_mysqlstandbyclusters.yaml b/config/crd/bases/shipstream.io_mysqlstandbyclusters.yaml index bd576ff..8e88993 100644 --- a/config/crd/bases/shipstream.io_mysqlstandbyclusters.yaml +++ b/config/crd/bases/shipstream.io_mysqlstandbyclusters.yaml @@ -5992,6 +5992,14 @@ spec: format: int64 minimum: 0 type: integer + readOnlyMaxLagSeconds: + description: |- + ReadOnlyMaxLagSeconds is the maximum lag for a read-only site's + client-facing endpoint. Nil inherits MaxLagSeconds; explicit zero + requires the reader to be fully caught up. + format: int64 + minimum: 0 + type: integer type: object restoreInPlace: description: |- @@ -6229,6 +6237,14 @@ spec: description: Annotations are additional annotations applied to every Service managed by this failover group. type: object + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy controls external traffic routing for Services + whose effective type is NodePort or LoadBalancer. + enum: + - Cluster + - Local + type: string type: description: 'Type is the Kubernetes Service type. Default: ClusterIP' @@ -6357,6 +6373,12 @@ spec: description: LBIP is the load balancer IP for DNS failover. minLength: 1 type: string + mysqlConf: + additionalProperties: + type: string + description: MysqlConf overrides group-level MySQL settings + for this site. + type: object name: description: |- Name is the site identifier (e.g. "iad", "pdx", "lhr"). @@ -6434,7 +6456,41 @@ spec: enum: - primary-candidate - dr-only + - read-only type: string + serviceTemplate: + description: |- + ServiceTemplate overrides the group Service template for this site's + client-facing MySQL Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are merged over group Service + annotations for this site. + type: object + externalTrafficPolicy: + description: ExternalTrafficPolicy overrides the + group policy. + enum: + - Cluster + - Local + type: string + nodePort: + description: NodePort requests a specific port for + this site's MySQL endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: + description: Type overrides the group Service type. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object storage: description: Storage configures the persistent volume for this site. @@ -6471,12 +6527,15 @@ spec: minLength: 1 type: string required: - - lbIP - name - storage - - taintNodeSelector - zone type: object + x-kubernetes-validations: + - message: taintNodeSelector and lbIP are required unless + role is read-only + rule: self.role == 'read-only' || (has(self.taintNodeSelector) + && has(self.lbIP)) maxItems: 16 minItems: 2 type: array @@ -6574,6 +6633,32 @@ spec: rule: '!has(self.splitBrainPolicy) || !has(self.splitBrainPolicy.sitePriorities) || self.splitBrainPolicy.sitePriorities.all(p, self.sites.exists(s, s.name == p && s.role == ''primary-candidate''))' + - message: serviceTemplate.externalTrafficPolicy requires serviceTemplate.type + NodePort or LoadBalancer + rule: '!has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || (has(self.serviceTemplate.type) && self.serviceTemplate.type + in [''NodePort'', ''LoadBalancer''])' + - message: site serviceTemplate.externalTrafficPolicy requires + an effective type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate) && has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' + - message: site serviceTemplate.nodePort requires an effective + type of NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(s.serviceTemplate) || !has(s.serviceTemplate.nodePort) + || ((has(s.serviceTemplate.type) ? s.serviceTemplate.type + : (has(self.serviceTemplate) && has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' + - message: inherited externalTrafficPolicy requires each effective + site service type to be NodePort or LoadBalancer + rule: 'self.sites.all(s, !has(self.serviceTemplate) || !has(self.serviceTemplate.externalTrafficPolicy) + || ((has(s.serviceTemplate) && has(s.serviceTemplate.type) + ? s.serviceTemplate.type : (has(self.serviceTemplate.type) + ? self.serviceTemplate.type : ''ClusterIP'')) in [''NodePort'', + ''LoadBalancer'']))' required: - name - spec diff --git a/docs/docs/app-integration.mdx b/docs/docs/app-integration.mdx index 7c992f3..d2f0164 100644 --- a/docs/docs/app-integration.mdx +++ b/docs/docs/app-integration.mdx @@ -78,8 +78,23 @@ For a `MysqlFailoverGroup` named `orders` in namespace `default`, the operator c |---|---|---| | `mysql-orders-primary` | `mysql-orders-primary.default.svc.cluster.local:3306` | Writes. Always points to the active primary. | | `mysql-orders-replicas` | `mysql-orders-replicas.default.svc.cluster.local:3306` | Reads. Points to healthy read replicas. | -| `mysql-orders-iad` | `mysql-orders-iad.default.svc.cluster.local:3306` | Direct access to the `iad` site instance. | -| `mysql-orders-pdx` | `mysql-orders-pdx.default.svc.cluster.local:3306` | Direct access to the `pdx` site instance. | +| `mysql-orders-iad` | `mysql-orders-iad.default.svc.cluster.local:3306` | Client-facing access to the `iad` site instance. | +| `mysql-orders-pdx` | `mysql-orders-pdx.default.svc.cluster.local:3306` | Client-facing access to the `pdx` site instance. | +| `mysql-orders-reader` | `mysql-orders-reader.default.svc.cluster.local:3306` | Health-gated site-specific read pool for a `read-only` site. | +| `mysql-orders--internal` | `mysql-orders--internal.default.svc.cluster.local` | Operator-only administrative route for MySQL `3306` and the sidecar. Do not use from applications. | + +Client-facing site Services expose only the named MySQL port. The sidecar is +not externally exposed and never receives a NodePort. Internal Services are +always `ClusterIP`, set `publishNotReadyAddresses: true`, and do not select on +the reader's `healthy` label. This preserves the operator's route for probes, +replication repair, clone, backup, and sidecar peer traffic even while the +reader pod is not ready or its application endpoint has been shed. + +When `spec.tls` is enabled, operator Go clients dial the internal Service but +continue verifying the existing client-facing site hostname as TLS +`ServerName`; existing certificates do not need to add the `-internal` name. +Strict hostname verification configured inside MySQL-native replication or +clone is separate and must trust the internal Service hostname. When `spec.dragonfly.enabled=true`, Bloodraven also owns the cache/session endpoints for the same failover group: @@ -334,3 +349,30 @@ A replica is considered healthy when: - Its replication lag is within `spec.replication.maxLagSeconds` If no healthy replicas exist, the Service will have no endpoints. Applications reading from the replicas Service should handle this by falling back to the primary or surfacing an error. + +## Site-specific reader pools + +A site with `role: read-only` is a non-promotable follower intended for a +local application read pool. Connect to its client-facing site Service, for +example `mysql-orders-reader.default.svc.cluster.local:3306`. Never send +writes to it: readers are held in `super_read_only`, are excluded from +promotion and active DNS, and do not participate in application node taints. + +The reader Service publishes an endpoint only when the latest debounced +topology snapshot confirms all of the following: + +- MySQL is read-only. +- Both replication I/O and SQL threads are healthy. +- `sourceConvergenceState` is `Converged` and the canonical `Source_Host` + directly names the uniquely confirmed active primary. +- Replication lag is known and is less than or equal to + `spec.replication.readOnlyMaxLagSeconds`. When omitted, this inherits + `maxLagSeconds`; explicit zero means only zero lag is accepted. + +If any check fails, Kubernetes removes the reader from the client Service at +the normal topology poll cadence. Existing connections are not an +operator-managed proxy and are not forcibly killed; applications must handle +connection errors and retry. Direct pod access and the internal Service remain +available to the operator for diagnosis and recovery. Reader failures remain +visible in site status and metrics but do not set the failover group's shared +`Ready=False` or `Degraded=True` conditions. diff --git a/docs/docs/backup-overview.mdx b/docs/docs/backup-overview.mdx index 038dad0..3691eee 100644 --- a/docs/docs/backup-overview.mdx +++ b/docs/docs/backup-overview.mdx @@ -25,6 +25,16 @@ Bloodraven backs up MySQL with mysqlsh `util.dumpInstance()` and restores with ` 4. Verify the backup with `MysqlBackupVerification`. 5. Document the restore path and test it on a non-production group. +## Backup source eligibility + +Bloodraven prefers a healthy replica within the configured backup lag +threshold and falls back to the confirmed active primary. A healthy +`dr-only` follower remains eligible. A `read-only` site is never selected as +an automatic backup source, and setting `sourceSiteOverride` or +`kubectl bloodraven backup --source-site` to a reader is rejected. This keeps +application read pools and backup load separate; reclassify the site only if +you intend to change its operational role. + ## Start here - [S3 Backups](./backup-s3.mdx) for object storage. diff --git a/docs/docs/configuration.mdx b/docs/docs/configuration.mdx index 835f769..1b328c9 100644 --- a/docs/docs/configuration.mdx +++ b/docs/docs/configuration.mdx @@ -32,6 +32,74 @@ Bloodraven has three layers: | Failover group | `MysqlFailoverGroup` | Database/platform team | | Application access | MySQL Services, optional Dragonfly Service, Secrets, DNS name | Application team with platform guardrails | +## MySQL configuration precedence + +Bloodraven renders a separate deterministic MySQL ConfigMap for every site. +The effective option order is: + +1. Bloodraven built-in defaults. +2. Normalized `spec.mysqlConf` group overrides. +3. Normalized `spec.sites[].mysqlConf` overrides for that site. +4. Bloodraven-owned replication safety settings, reasserted last. + +Option names normalize `_` to `-` before every merge. For example, +`max_connections` and `max-connections` address the same effective key; do +not define both spellings in one map and expect two settings. A site spelling +overrides the equivalent normalized group key. Final keys are sorted, so +semantically equivalent maps render and hash identically. + +The operator always owns `gtid-mode`, `enforce-gtid-consistency`, `log-bin`, +`log-replica-updates`, `skip-replica-start`, and clone plugin loading. User +values cannot weaken these replication invariants. + +Each Deployment mounts `mysql---config`. A rendered-config hash, +not the raw maps, drives drift: changing or deleting a site override rolls +that site while leaving unaffected sites alone. Ordered updates restart every +drifted non-active follower sequentially after checking read-only state, +direct source, and replication threads. If the active site also drifted, a +healthy promotable standby takes over before the old active is restarted. A +reader-only or other non-active-only change never causes failover, and neither +readers nor `dr-only` sites are chosen as handoff targets. + +Upgrades from the legacy group ConfigMap are level-driven. Bloodraven creates +all per-site maps first, rolls live Deployment references one site at a time, +and deletes the legacy map only after every Deployment references its expected +site map. Reconciliation can resume safely after an operator restart at any +point in this migration. + +## Service inheritance and isolation + +`spec.serviceTemplate` defaults the client-facing site Services and the +`-primary` and `-replicas` Services. A site's +`spec.sites[].serviceTemplate` overrides only that site's client Service: + +- Non-zero site `type`, `externalTrafficPolicy`, and `nodePort` values override + inherited values. +- Group and site annotations merge, with site keys winning. Site annotations + never leak to siblings, `-primary`, `-replicas`, or internal Services. +- `externalTrafficPolicy` is legal only when the effective Service type is + `NodePort` or `LoadBalancer`. The group value also applies to primary and + replicas Services when their type supports it. +- `nodePort` is site-only, legal only for `NodePort` or `LoadBalancer`, and + applies only to the named MySQL port. Kubernetes admission enforces the + cluster's configured allocation range. + +Omitting a site NodePort preserves its compatible allocated MySQL NodePort +while the Service remains external. Switching between `NodePort` and +`LoadBalancer` preserves that compatible allocation; switching to `ClusterIP` +clears external-only fields. Bloodraven preserves Kubernetes-assigned Service +identity and annotations it does not own. + +For every site, Bloodraven also creates +`mysql---internal`. It is always `ClusterIP`, ignores all user +Service templates, publishes not-ready addresses, and exposes MySQL plus the +sidecar for operator, replication, clone, backup, and peer traffic. The +client-facing Service exposes MySQL only. When Go clients dial the internal +Service with `spec.tls`, Bloodraven retains the client-facing site hostname as +the TLS `ServerName`, so existing certificates do not need another DNS name. +Operators that enable strict hostname verification inside MySQL-native clone +or replication must account for the internal Service hostname separately. + ## Recommended production baseline - Use `spec.credentials` instead of legacy `spec.secretName`. diff --git a/docs/docs/crd-reference.mdx b/docs/docs/crd-reference.mdx index 03c7635..ab7023b 100644 --- a/docs/docs/crd-reference.mdx +++ b/docs/docs/crd-reference.mdx @@ -49,7 +49,7 @@ Review runbooks before changing `spec.restoreInPlace`, restore-related `confirm` |---|---|---|---|---| | `image` | string | No | `mysql:9.7` | MySQL container image | | `sidecarImage` | string | No | `ghcr.io/shipstream/bloodraven-sidecar:0.1.6` | Bloodraven sidecar container image | -| `sites` | []SiteSpec | Yes | -- | Site definitions. At least two sites with `role: primary-candidate`; additional `dr-only` sites may be appended. MinItems=2, MaxItems=16. | +| `sites` | []SiteSpec | Yes | -- | Site definitions. At least two sites with `role: primary-candidate`; additional `dr-only` and `read-only` followers may be appended. MinItems=2, MaxItems=16. | | `secretName` | string | No | -- | **(Legacy)** Name of the Secret containing MySQL credentials (key: `dsn`). Mutually exclusive with `credentials`. | | `credentials` | CredentialsSpec | No | -- | Per-role MySQL credential management. The operator creates MySQL users with role-appropriate privileges and rotates passwords when secrets change. Mutually exclusive with `secretName`. | | `dns` | DNSSpec | Yes | -- | DNS configuration for traffic steering via external-dns | @@ -68,7 +68,7 @@ Review runbooks before changing `spec.restoreInPlace`, restore-related `confirm` | `cloneTimeout` | int | No | `3600` | Timeout in seconds for initial data clone operations | | `podLabels` | map[string]string | No | -- | Additional labels applied to every MySQL pod. Operator labels take precedence on conflict. | | `podAnnotations` | map[string]string | No | -- | Additional annotations applied to every MySQL pod. Operator annotations take precedence on conflict. | -| `serviceTemplate` | ServiceTemplate | No | -- | Customizes the Services created by the operator | +| `serviceTemplate` | ServiceTemplate | No | -- | Defaults the client-facing site, primary, and replicas Services. Internal per-site Services ignore this template and remain ClusterIP. | | `extraContainers` | []Container | No | -- | Additional containers injected into every MySQL pod (e.g. exporters) | | `extraInitContainers` | []Container | No | -- | Additional init containers injected after the operator's built-in init container | | `backup` | BackupSpec | No | -- | Backup configuration (profiles, schedules, retry, PITR, encryption, security contexts). See [BackupSpec](#backupspec). | @@ -84,12 +84,33 @@ Each entry in `spec.sites` defines one site in the failover group. | Field | Type | Required | Default | Description | |---|---|---|---|---| | `name` | string | Yes | -- | Unique site identifier (e.g. `iad`, `pdx`). Used in Service names and labels. MaxLength=253. | -| `role` | string | No | `primary-candidate` | Promotion eligibility: `primary-candidate` (can be auto-promoted on failover) or `dr-only` (passive replica; never auto-promoted). At least two `primary-candidate` sites are required. | +| `role` | string | No | `primary-candidate` | `primary-candidate` can be promoted; `dr-only` and `read-only` are never promoted. A reader provides a health-gated site-specific read pool. At least two `primary-candidate` sites are required. | | `zone` | string | Yes | -- | Availability zone or region for pod scheduling | -| `taintNodeSelector` | map[string]string | Yes | -- | Label selector used to find nodes that receive this group's `db-readonly` taint for this site. Use group-scoped labels such as `shipstream.io/failover-group.orders=true` and `shipstream.io/site.orders=iad` for shared nodes. | -| `lbIP` | string | Yes | -- | Load balancer IP address. Used for DNS A-record steering on failover. | +| `taintNodeSelector` | map[string]string | Conditional | -- | Required and non-empty for `primary-candidate` and `dr-only`; optional for `read-only`. Selects nodes that receive this group's `db-readonly` taint. Accepted but ignored on readers, which never cause taint operations. | +| `lbIP` | string | Conditional | -- | Required and non-empty for `primary-candidate` and `dr-only`; optional for `read-only`. Used for active-primary DNS steering. Accepted but ignored on readers, which are never DNS targets. | | `storage` | StorageSpec | Yes | -- | Persistent storage configuration for this site | | `resources` | ResourceRequirements | No | -- | Compute resources (requests/limits) for the MySQL container at this site | +| `mysqlConf` | map[string]string | No | -- | Site-specific MySQL options applied after normalized group overrides. Changing this map rolls only affected sites unless other drift is pending. | +| `serviceTemplate` | SiteServiceTemplate | No | -- | Scalar and annotation overrides for this site's client-facing MySQL Service. Does not affect sibling, primary, replicas, or internal Services. | + +Reader manifests may retain legacy `lbIP` and `taintNodeSelector` values for +compatibility, but Bloodraven ignores them. Changing a reader to +`primary-candidate` or `dr-only` is rejected until both fields are present. +A reader does not satisfy the two-candidate minimum and cannot appear in +`splitBrainPolicy.sitePriorities`. + +### Admission combinations + +| Combination | Admitted? | +|---|---| +| Two `primary-candidate` sites, each with non-empty `lbIP` and `taintNodeSelector` | Yes | +| Two candidates plus a reader with both placement fields omitted | Yes | +| Reader with either or both legacy placement fields supplied | Yes; supplied values are ignored | +| Omitted/default role, explicit candidate, or `dr-only` with either placement field omitted | No | +| Fewer than two candidates, or a reader/DR site in `sitePriorities` | No | +| `externalTrafficPolicy` on an effective `ClusterIP` Service | No | +| Site `nodePort` on an effective `ClusterIP` Service | No | +| ETP on `NodePort`/`LoadBalancer`, or site NodePort in 1-65535 on those types | Yes; Kubernetes then applies its cluster-specific NodePort range | ### StorageSpec @@ -124,6 +145,7 @@ Each entry in `spec.sites` defines one site in the failover group. | Field | Type | Required | Description | |---|---|---|---| | `maxLagSeconds` | int | No | Maximum acceptable replication lag in seconds before the replica is considered unhealthy | +| `readOnlyMaxLagSeconds` | int | No | Maximum lag for a reader's client endpoint. Omitted inherits `maxLagSeconds` (including its 300-second effective fallback); explicit `0` requires zero reported lag. | ### SplitBrainPolicySpec @@ -142,7 +164,22 @@ full description of semantics and tradeoffs. | Field | Type | Required | Description | |---|---|---|---| | `type` | string | No | Kubernetes Service type: `ClusterIP` (default), `LoadBalancer`, or `NodePort` | -| `annotations` | map[string]string | No | Additional annotations applied to every Service managed by the operator | +| `externalTrafficPolicy` | string | No | `Cluster` or `Local`. Valid only when the effective Service type is `NodePort` or `LoadBalancer`. Applied to site, primary, and replicas Services when supported. | +| `annotations` | map[string]string | No | Default annotations for client-facing site, primary, and replicas Services. Not applied to internal Services. | + +### SiteServiceTemplate (`spec.sites[].serviceTemplate`) + +| Field | Type | Required | Description | +|---|---|---|---| +| `type` | string | No | Site Service override: `ClusterIP`, `LoadBalancer`, or `NodePort`. Omitted inherits the group type. | +| `externalTrafficPolicy` | string | No | Site override: `Cluster` or `Local`. Legal only when the effective site type is `NodePort` or `LoadBalancer`. | +| `nodePort` | int | No | Requested NodePort for this site's named `mysql` port, range 1-65535. Legal only for effective type `NodePort` or `LoadBalancer`; cluster admission enforces its configured allocation range. | +| `annotations` | map[string]string | No | Merged over group annotations for this site only; site keys win. | + +The operator preserves Kubernetes-assigned Service identity and compatible +allocated NodePorts across reconciles and external-type transitions. Moving a +site Service to `ClusterIP` clears external-only fields. The sidecar port is +never exposed through a client Service or assigned a NodePort. ### CredentialsSpec @@ -375,6 +412,20 @@ spec: limits: cpu: "4" memory: "16Gi" + - name: reader + role: read-only + zone: us-east-1b + mysqlConf: + innodb_buffer_pool_size: "4G" + serviceTemplate: + type: LoadBalancer + externalTrafficPolicy: Local + nodePort: 32006 + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + storage: + storageClassName: fast-ssd + size: 100Gi secretName: mysql-credentials sidecarResources: requests: @@ -402,6 +453,7 @@ spec: sync_binlog: "1" replication: maxLagSeconds: 300 + readOnlyMaxLagSeconds: 30 updateStrategy: OrderedUpdate dragonfly: enabled: true @@ -416,7 +468,8 @@ spec: prometheus.io/scrape: "true" prometheus.io/port: "9104" serviceTemplate: - type: ClusterIP + type: LoadBalancer + externalTrafficPolicy: Local annotations: service.beta.kubernetes.io/aws-load-balancer-internal: "true" extraContainers: @@ -525,6 +578,9 @@ The operator writes the following status fields. | `replicating` | bool | Whether this site is replicating from another site | | `secondsBehindSource` | int | Replication lag in seconds (only set when replicating) | | `gtidExecuted` | string | GTID set executed on this site | +| `sourceHost` | string | Source hostname reported by MySQL. Followers are expected to name the active primary's internal site Service. | +| `sourceConvergenceState` | string | `Converged`, `Pending`, or `Blocked`. Generic direct-source state for every follower; independent of old-primary recovery state. | +| `sourceConvergenceReason` | string | Stable reason: `DirectSource`, `SourceMismatch`, `ProbeFailed`, `MutationFailed`, or `GTIDDiverged`. Detailed errors remain in structured logs. | | `recoveryState` | string | Old-primary recovery state: empty (no recovery needed), `RecoveryInProgress` (the operator is reconfiguring or waiting for the site to stabilize as a replica), or `RecoveryBlocked` (divergent transactions detected — trigger a reclone to recover) | | `divergentGtid` | string | GTID set of transactions on this site that diverge from the current primary. Populated when `recoveryState` is `RecoveryBlocked`. | | `divergentTransactionCount` | int | Number of divergent transactions. Populated when `recoveryState` is `RecoveryBlocked`. | @@ -533,8 +589,8 @@ The operator writes the following status fields. | Type | Meaning | |---|---| -| `Ready` | `True` when the failover group has one writable primary and all replicas are healthy | -| `Degraded` | `True` when replication is broken, lag exceeds `maxLagSeconds`, or a site is unreachable | +| `Ready` | `True` when there is exactly one confirmed writable `primary-candidate` and every non-reader follower has healthy direct replication. Reader reachability, lag, threads, and source state do not change this group condition. | +| `Degraded` | `True` when core candidate/DR replication is broken or source-mismatched, lag exceeds `maxLagSeconds`, or a non-reader site is unreachable. Reader failures are excluded and remain visible per site and in metrics. | | `RecoveryPending` | `True` while old-primary recovery is running (`RecoveryInProgress`) or when an old primary has divergent transactions (`DivergentTransactions`). Use the reclone annotation only for `DivergentTransactions`. | | `Bootstrapping` | `True` when a clone operation is in progress (fresh-deploy, auto-clone, or reclone). Reason indicates the phase. | @@ -591,6 +647,18 @@ status: replicating: true secondsBehindSource: 0 gtidExecuted: "uuid:1-100" + sourceHost: mysql-orders-iad-internal.orders.svc.cluster.local + sourceConvergenceState: Converged + sourceConvergenceReason: DirectSource + - name: reader + state: read-only + lastSeen: "2025-01-01T00:00:00Z" + replicating: true + secondsBehindSource: 0 + gtidExecuted: "uuid:1-100" + sourceHost: mysql-orders-iad-internal.orders.svc.cluster.local + sourceConvergenceState: Converged + sourceConvergenceReason: DirectSource conditions: - type: Ready status: "True" diff --git a/docs/docs/examples.mdx b/docs/docs/examples.mdx index a3bd75e..6cbdc4b 100644 --- a/docs/docs/examples.mdx +++ b/docs/docs/examples.mdx @@ -11,7 +11,7 @@ Complete example manifests live in the repository `examples/` directory so they | Example | File | |---|---| -| Minimal failover group | [`examples/minimal-failovergroup.yaml`](https://github.com/ShipStream/bloodraven/blob/main/examples/minimal-failovergroup.yaml) | +| Minimal failover group with two candidates and a placement-field-free reader | [`examples/minimal-failovergroup.yaml`](https://github.com/ShipStream/bloodraven/blob/main/examples/minimal-failovergroup.yaml) | | Per-role credentials | [`examples/per-role-credentials.yaml`](https://github.com/ShipStream/bloodraven/blob/main/examples/per-role-credentials.yaml) | | TLS-enabled failover group | [`examples/tls-enabled-failovergroup.yaml`](https://github.com/ShipStream/bloodraven/blob/main/examples/tls-enabled-failovergroup.yaml) | | S3 backups with scheduled verification | [`examples/s3-backups-with-verification.yaml`](https://github.com/ShipStream/bloodraven/blob/main/examples/s3-backups-with-verification.yaml) | diff --git a/docs/docs/failover.mdx b/docs/docs/failover.mdx index 8f765ea..d1466c1 100644 --- a/docs/docs/failover.mdx +++ b/docs/docs/failover.mdx @@ -57,6 +57,15 @@ After updating individual site states, the operator evaluates the pair together: The operator only takes automatic action for the failover case (and, opt-in, for split brain). All other anomalous states require human investigation. +The table is the two-candidate core reduced to two columns. In an N-site +group, only `primary-candidate` sites can be the active site, promotion target, +or split-brain winner. `dr-only` and `read-only` sites are non-promotable. A +writable non-promotable site is an anomaly and is fenced on every poll; a +reader is fenced even when it is the sole writable site. Reader outages and +replication failures remain visible per site but are excluded from group +readiness and degradation calculations. Readers never trigger node taint +changes or become active DNS targets. + **Exception — re-asserting a fenced promoted primary.** A freshly promoted primary can be fenced back to read-only by its own sidecar: the sidecar's fencing lease may still be stale when the promotion lands (for example, the operator restarted after a full-site outage and promoted before its auxiliary Service endpoint became Ready, so the sidecar's operator probes kept failing while the operator was already driving MySQL). That leaves every site reachable and read-only — a state the table above refuses to touch, because without history it is indistinguishable from a fresh-start condition that needs human input. The operator, however, does have history: `status.lastFailoverTarget` names the site it made authoritative. When **all** of the following hold, the operator restores writability on that site instead of alerting and waiting: @@ -98,6 +107,39 @@ When the operator decides to fail over to a candidate site, it executes these st - Taint old active site nodes: `shipstream.io/db-readonly-=true:NoExecute` - Untaint new active site nodes: remove the `shipstream.io/db-readonly-` taint +10. **Converge every follower's source** + - Candidate, `dr-only`, and `read-only` followers are verified against the + newly confirmed active primary + - Each follower must replicate directly from the primary; replication + chains are not accepted as converged + +## Direct-source convergence + +Source convergence runs after replica status collection and outside the +promotion sequence. It is therefore able to repair a healthy wrong source +after an operator restart even when no failover history exists. Mutation is +allowed only when there is exactly one writable `primary-candidate`, no second +writable site, and no bootstrap, update, restore, topology freeze, pending +promotion, planned failover, or split brain in flight. + +For a wrong non-empty source, Bloodraven: + +1. Verifies that the active primary's `GTID_EXECUTED` contains the follower's + executed set. +2. Runs `STOP REPLICA` and repeats both GTID reads and the containment check, + closing the race where the SQL applier advances after the first check. +3. Runs `CHANGE REPLICATION SOURCE TO` with the configured credentials and TLS + settings, without `RESET REPLICA ALL`. +4. Runs `START REPLICA` and boundedly verifies the direct canonical hostname + and both replication threads. + +If containment fails, status becomes `Blocked/GTIDDiverged` and no unsafe +repoint occurs. A post-STOP containment failure leaves replication stopped. +Other bounded failures remain `Pending/MutationFailed` for a later safe retry. +This generic state is recorded in `sourceHost`, `sourceConvergenceState`, and +`sourceConvergenceReason`; it does not replace old-primary +`recoveryState`/`divergentGtid` reporting. + ## Dragonfly during failover When `spec.dragonfly.enabled=true`, Dragonfly follows the MySQL failover group but remains best-effort cache/session state, not durable data. @@ -165,7 +207,7 @@ To recover a divergent site: kubectl annotate mysqlfailovergroup bloodraven.shipstream.io/reclone-site=: ``` -3. The operator validates that the prefix matches the observed `divergentGtid` — a mismatch is rejected with a `RecloneRejected` Warning Event, so a fat-fingered site name can't destroy the wrong replica. When a site has **no** `divergentGtid` (cold reclone: PVC loss, manual rebuild), the bare form `reclone-site=` is still accepted. +3. The operator validates that the prefix matches the observed `divergentGtid` — a mismatch is rejected with a `RecloneRejected` Warning Event, so a fat-fingered site name can't destroy the wrong replica. When a site has **no** `divergentGtid` (cold reclone: PVC loss, manual rebuild), use `kubectl bloodraven reclone --cold` to provide the required destructive confirmation. 4. The operator runs `CLONE INSTANCE` on the target site, replacing all data with a fresh copy from the current primary. A `RecloneRequested` Event marks the start. See [Recovering a divergent old primary](./operations.mdx#recovering-a-divergent-old-primary) for the full procedure. @@ -243,7 +285,7 @@ The cooldown timer resets after each failover. The last failover time is recorde ## Ordered updates -When `spec.updateStrategy` is set to `OrderedUpdate`, spec changes (such as a new `image` or resource adjustments) are rolled out with zero downtime: +When `spec.updateStrategy` is set to `OrderedUpdate`, spec changes (such as a new `image`, effective per-site MySQL config, or resource adjustments) are rolled out with zero downtime: ```mermaid sequenceDiagram @@ -268,6 +310,14 @@ This sequence ensures: - Replication is healthy before each transition - At most one site is unavailable at any time +For groups with more than two sites, every drifted non-active follower is +updated sequentially and must be read-only with a direct healthy source before +and after restart. If the active site has no drift, the rollout completes +without failover; this is the normal reader-only configuration path. If the +active site is drifted, only a healthy `primary-candidate` standby may receive +the handoff. A reader or `dr-only` follower is never promoted to facilitate an +update, and failed or unprocessed drift remains queued for a later reconcile. + Without `OrderedUpdate`, both sites are updated simultaneously, which may cause brief downtime if both pods restart at the same time. For MySQL image changes, see the [Upgrade and version-skew policy](./upgrade-policy.mdx). The replica-first ordering above is the MySQL-required direction for a rolling version upgrade. diff --git a/docs/docs/known-limitations.mdx b/docs/docs/known-limitations.mdx index d3329b3..c3a7499 100644 --- a/docs/docs/known-limitations.mdx +++ b/docs/docs/known-limitations.mdx @@ -31,6 +31,13 @@ matches your expectations. - If you require synchronous commit semantics or quorum-based zero RPO on primary loss, Bloodraven is the wrong tool; see [Why not Group Replication?](./why-not-group-replication.mdx). +- Read-only sites are full MySQL replicas. Filtered or partial replication, + replication proxying, and query routing through an operator-managed proxy + are not implemented. +- Clone and reclone copy from the active primary at full speed. Bloodraven + does not throttle clone traffic, so adding or rebuilding a large reader can + consume primary network, storage, and CPU capacity. Schedule the operation + and monitor the donor accordingly. ## Operator availability @@ -65,6 +72,8 @@ matches your expectations. into a separate Kubernetes cluster from the source backup archive. See [Multi-cluster DR](./multi-cluster-dr.mdx) for the end-to-end runbook using `spec.initFromBackup` with optional PITR replay. +- `read-only` sites cannot be selected for scheduled, automatic, or + source-overridden backups. Healthy `dr-only` followers remain eligible. ## Dragonfly co-management @@ -93,8 +102,9 @@ matches your expectations. - Grafana dashboards and metrics are shipped, but PrometheusRule alert examples are not yet packaged as first-class chart artifacts. -- There is no `kubectl bloodraven` plugin yet. Use annotations, - `kubectl get`, and `kubectl describe` for operations. +- The `kubectl bloodraven` plugin wraps supported day-2 operations, but it + remains a thin API client; the operator performs the authoritative safety + checks. ## External dependencies diff --git a/docs/docs/kubectl-plugin.mdx b/docs/docs/kubectl-plugin.mdx index 0d49909..1db462c 100644 --- a/docs/docs/kubectl-plugin.mdx +++ b/docs/docs/kubectl-plugin.mdx @@ -51,7 +51,7 @@ you'd hit with raw `kubectl get/annotate/create`. |---|---|---| | `status [group]` | One-shot health view for one group or every group in the namespace | `kubectl get mysqlfailovergroup -o wide` + `kubectl describe` | | `promote ` | Apply the `planned-failover` annotation with optional `--max-lag-wait` override | `kubectl annotate mysqlfailovergroup bloodraven.shipstream.io/planned-failover=:maxLagWait=...` | -| `reclone ` | Re-clone a divergent site; auto-fills the GTID prefix from `status.sites[].divergentGtid` | `kubectl annotate mysqlfailovergroup bloodraven.shipstream.io/reclone-site=:` | +| `reclone ` | Re-clone a divergent follower, or cold-reclone a candidate, DR site, or reader with `--cold`; auto-fills the GTID prefix when present | `kubectl annotate mysqlfailovergroup bloodraven.shipstream.io/reclone-site=:` | | `backup --profile ` | Create a `MysqlBackup` CR; optional `--source-site`, `--wait` | `kubectl create -f mysqlbackup.yaml` | | `verify-backup --profile ` | Create a `MysqlBackupVerification` CR; optional `--backup`, `--wait` | `kubectl create -f mysqlbackupverification.yaml` | @@ -133,6 +133,20 @@ kubectl bloodraven reclone orders iad -n orders --cold kubectl bloodraven reclone orders iad -n orders --gtid-prefix=a1b2c3d4 ``` +Readers use the same destructive cold-reclone workflow. The exact generic +form is: + +```bash +kubectl bloodraven reclone --cold +``` + +`--cold` is mandatory when no divergent GTID confirmation is available. It +generates the group-name confirmation token, and the operator still rejects +the active primary or an invalid target. A successful request wipes the +reader datadir, clones from the uniquely confirmed active primary, and then +waits for direct replication to converge. Clone traffic is unthrottled and +runs against the primary, so plan capacity before rebuilding a large reader. + ### backup ```bash @@ -151,6 +165,8 @@ The plugin validates that the profile exists in `spec.backup.profiles` and that any `--source-site` is in `spec.sites` before posting the CR — so a typo fails immediately instead of being deferred to the operator's reconcile loop. +The operator additionally rejects `read-only` sites as backup sources; +readers are ineligible both for automatic selection and explicit overrides. ### verify-backup @@ -201,3 +217,8 @@ summary at the end. progress on the server side; re-run `kubectl bloodraven status` to inspect the current phase. `--wait --timeout` is the polling budget, not the operator's budget. +* A site with `sourceConvergenceState: Blocked` and + `sourceConvergenceReason: GTIDDiverged` cannot be safely repointed. Compare + the follower and active-primary GTID sets, preserve any data that needs + review, then use the divergent-prefix reclone flow or the explicit reader + cold-reclone command above. diff --git a/docs/docs/log-schema.mdx b/docs/docs/log-schema.mdx index cbe982f..3d21a35 100644 --- a/docs/docs/log-schema.mdx +++ b/docs/docs/log-schema.mdx @@ -61,7 +61,7 @@ Sidecar records additionally carry: - Keys are `camelCase`. Common keys: `site`, `fg`, `error`, `peer`, `count`, `source`, `donor`, `recipient`. - Site identifiers (`site`, `oldPrimary`, `newPrimary`, `promotedSite`, `donor`, `recipient`, `activeSite`, `authoritativeActiveSite`) all carry the bare site name as defined in `spec.sites[].name`. -- GTID fields (`promotionGtid`, `divergentGtid`, `oldPrimaryGtid`, `newPrimaryGtid`) carry MySQL GTID-set strings exactly as MySQL returns them — never parsed or canonicalised. +- GTID fields (`promotionGtid`, `divergentGtid`, `oldPrimaryGtid`, `newPrimaryGtid`, `followerGtid`, `activeGtid`) carry MySQL GTID-set strings exactly as MySQL returns them — never parsed or canonicalised. - Counts (`count`, `divergentTransactions`, `attempt`, `maxRetries`) are JSON numbers, not strings. - Durations (`leaseTimeout`, `pollInterval`, `delay`) are emitted by `slog`'s default `time.Duration` rendering — currently a string like `"30s"`. Treat as opaque if you need to parse, prefer the metric of the same name. @@ -107,6 +107,25 @@ Fired after an emergency failover when the operator inspects the returning old p | INFO | `old primary recovery complete` | `site`, `source`, `fg` | Old primary is now replicating from the new primary. `source` is the new primary's host. | | ERROR | `old primary recovery failed` | `site`, `error`, `fg` | One step of the recovery sequence (fence / GTID query / `CHANGE REPLICATION SOURCE` / `START REPLICA`) returned an error. | +### Replication source convergence + +After topology changes, Bloodraven verifies that every follower replicates +directly from the uniquely confirmed active primary. These events cover +candidate, `dr-only`, and `read-only` followers; they are separate from the +old-primary recovery events above. + +| Level | `msg` | Fields | Notes | +|---|---|---|---| +| INFO | `replication source convergence started` | `site`, `activeSite`, `currentSource`, `expectedSource`, `fg` | A follower needs a source or thread-state correction and passed the initial mutation gates. | +| INFO | `replication source convergence complete` | `site`, `source`, `fg` | The canonical source is the active primary and both replication threads are running. | +| WARN | `replication source convergence blocked` | `site`, `activeSite`, `stage`, `followerGtid`, `activeGtid`, `fg` | GTID containment failed before or after stopping replication. No source change is issued. | +| ERROR | `replication source convergence failed` | `site`, `activeSite`, `stage`, `error`, `fg` | A bounded source mutation or verification attempt failed. The next poll can retry safely. | + +Stable `stage` values include `pre-stop-gtid`, `post-stop-gtid`, `stop`, +`change-source`, `start`, and `verify`. Use the status +`sourceConvergenceState` and `sourceConvergenceReason` for current state; use +these logs for the detailed failure and GTID evidence. + ### Bootstrap and reclone `starting bootstrap` is the single canonical event for "we are about to clone a replica". The `source` field disambiguates *why*: diff --git a/docs/docs/monitoring-prometheus.mdx b/docs/docs/monitoring-prometheus.mdx index c6e8916..73fde2c 100644 --- a/docs/docs/monitoring-prometheus.mdx +++ b/docs/docs/monitoring-prometheus.mdx @@ -87,6 +87,34 @@ curl http://localhost:8080/metrics | grep '^bloodraven_' In Prometheus, check **Status > Targets** for the `bloodraven` job or ServiceMonitor-generated target. +## Reader and source-convergence monitoring + +`bloodraven_replication_source_state{site,state}` is a state-set gauge for +every follower. It emits the bounded `state` values `converged`, `pending`, +and `blocked`; exactly one is `1` for a follower and the others are `0`. +Active primaries have no active source state. Combine it with +`bloodraven_replication_running{site,thread}` and +`bloodraven_replication_lag_seconds{site}` when alerting on a reader. + +Reader failures are deliberately isolated from the failover group's shared +`Ready` and `Degraded` conditions. A reader can be unreachable, lagging, or +source-blocked while the core candidate/DR topology remains Ready and not +Degraded. Alert on reader sites directly rather than inferring reader health +from group conditions. For example: + +```promql +bloodraven_replication_source_state{site="reader",state!="converged"} == 1 +``` + +```promql +bloodraven_replication_running{site="reader"} == 0 +or bloodraven_replication_lag_seconds{site="reader"} > 30 +``` + +Use a sustained `for` interval appropriate to the normal poll and recovery +cadence. A blocked source commonly needs GTID investigation and possibly a +cold reclone; a pending source may clear on a later bounded retry. + ## Alerts Keep alert rules with your platform monitoring stack. Alert names should link to [Alert To Runbook Map](./alert-runbook-map.mdx), and metric details live in [Monitoring Reference](./monitoring.mdx). diff --git a/docs/docs/monitoring.mdx b/docs/docs/monitoring.mdx index e59bda0..ff066e9 100644 --- a/docs/docs/monitoring.mdx +++ b/docs/docs/monitoring.mdx @@ -55,6 +55,7 @@ Metrics are served on `:8080/metrics` in standard Prometheus exposition format. | `bloodraven_websocket_connected_clients` | Gauge | -- | Number of currently connected WebSocket clients | | `bloodraven_replication_lag_seconds` | Gauge | `site` | Replication lag in seconds on the replica site. `-1` if lag is NULL (not replicating). Only present for replica sites. | | `bloodraven_replication_running` | Gauge | `site`, `thread` | Whether a replication thread is running (`1`=yes, `0`=no). Thread is `io` or `sql`. Only present for replica sites. | +| `bloodraven_replication_source_state` | Gauge | `site`, `state` | Direct-source convergence state set for each follower. `state` is one of `converged`, `pending`, or `blocked`; one series is `1` and the others are `0`. Cleared for the active primary. | | `bloodraven_site_state` | Gauge | `site`, `state` | Current site state as a state-set: `1` for the current state, `0` for others. State is `writable`, `read-only`, `unreachable`, or `unknown`. | | `bloodraven_divergent_transactions` | Gauge | `site` | Number of divergent transactions on a site pending recovery after emergency failover. `0` when healthy. Non-zero means the site has committed transactions that never replicated to the current primary. | | `bloodraven_archiver_upload_failures` | Gauge | `namespace`, `group`, `site` | Cumulative PITR archiver upload failures reported by the per-site sidecar. Monotonic except across sidecar restarts — use `increase()` / `rate()` in dashboards. | diff --git a/docs/docs/multi-site.mdx b/docs/docs/multi-site.mdx index 541499b..5ae7be1 100644 --- a/docs/docs/multi-site.mdx +++ b/docs/docs/multi-site.mdx @@ -15,11 +15,21 @@ failover: |------|----------------|-------------| | `primary-candidate` (default) | yes | active/standby pair inside a region | | `dr-only` | no | cross-region follower, kept for disaster recovery | +| `read-only` | no | site-specific application read pool | The spec must contain at least **two** `primary-candidate` sites so the operator always has a promotion target for the current primary. Any -number of `dr-only` sites may be appended for cross-region DR or -read-only regional fan-out. +number of `dr-only` or `read-only` sites may be appended. A reader does +not count toward the two-candidate minimum. + +Bloodraven uses these terms consistently: + +| Term | Meaning | +|---|---| +| Reader | A site with effective role `read-only`. | +| Follower | Any non-active site with replication metadata, including candidate, DR, and reader sites. | +| Standby | A healthy, promotable `primary-candidate` follower. | +| Active primary | The unique observed writable site, which must have role `primary-candidate`. | ## Replication topology @@ -31,13 +41,22 @@ chains. primary (iad, primary-candidate) / | \ / | \ - (pdx) (fra) (syd) - primary-candidate primary-candidate dr-only + (pdx) (fra) (reader) + primary-candidate dr-only read-only ``` -During failover, the newly promoted primary becomes the source for -every remaining replica (including `dr-only` sites) via a single -`CHANGE REPLICATION SOURCE` per replica. +During planned or emergency failover, the newly promoted primary becomes the +direct source for every remaining follower, including `dr-only` and +`read-only` sites. Bloodraven also detects a healthy-looking follower that +still points through another replica after restart or topology drift. + +Before repointing, the operator requires exactly one writable promotable +primary and checks that its `GTID_EXECUTED` contains the follower's executed +set. It repeats containment after `STOP REPLICA` to close the applier race, +then uses `CHANGE REPLICATION SOURCE TO` and `START REPLICA` without resetting +replication metadata. Divergence is reported as +`sourceConvergenceState: Blocked` with reason `GTIDDiverged`; Bloodraven does +not silently abandon follower transactions. ## Failover target selection @@ -57,7 +76,12 @@ best candidate using this order: 3. **Declared order.** Final tiebreaker for replicas that share GTID sets and are not named in the priority list. -`dr-only` sites are never auto-promoted. +`dr-only` and `read-only` sites are never auto-promoted, selected by planned +failover, used as active DNS targets, or chosen as clone donors. A reader that +becomes writable is fenced on every topology poll, even if it is the only +writable site. Readers also never apply or remove node taints; supplied +`lbIP` and `taintNodeSelector` fields are accepted for manifest compatibility +but ignored. If *no* `primary-candidate` replica is reachable when the primary fails, the operator emits a `NoPrimary` alert and takes no action. @@ -143,10 +167,9 @@ Failover behaviour: ## Sidecar peer awareness -Each pod's sidecar is given the list of peer sidecar addresses (every -non-self site) via the `PEER_ADDRESSES` env var — a comma-separated -list that the operator populates from `spec.sites[]` at reconcile -time. The sidecar tracks per-peer liveness and only self-fences when +Each pod's sidecar is given the internal Service addresses for every +non-self site, including `dr-only` sites and readers, via the +`PEER_ADDRESSES` env var. The sidecar tracks per-peer liveness and only self-fences when the operator **and every peer** are unreachable beyond `spec.sidecar.leaseTimeout`. A single reachable peer is enough to keep the primary writable. @@ -156,22 +179,66 @@ the primary can still reach at least one peer is preserved as a legitimate-writes window rather than collapsing to a self-fenced outage. +Reader peers provide connectivity and relay fresh operator-authoritative +active-site observations; they gain no promotion or primary authority. A +reachable peer without fresh authoritative topology can still suppress the +lease-only all-peers-unreachable fence. This is retained compatibility +behavior, not a quorum guarantee. + +## Example: two candidates and a reader + +The reader omits candidate/DR-only placement fields, uses a lower endpoint +lag threshold, and customizes only its own client Service and MySQL config: + +```yaml +spec: + replication: + maxLagSeconds: 300 + readOnlyMaxLagSeconds: 30 + sites: + - name: iad + role: primary-candidate + zone: iad-1a + taintNodeSelector: + shipstream.io/site.orders: iad + lbIP: 10.0.0.10 + storage: {storageClassName: gp3, size: 500Gi} + - name: pdx + role: primary-candidate + zone: pdx-1a + taintNodeSelector: + shipstream.io/site.orders: pdx + lbIP: 10.1.0.10 + storage: {storageClassName: gp3, size: 500Gi} + - name: reader + role: read-only + zone: iad-1b + mysqlConf: + innodb_buffer_pool_size: 4G + serviceTemplate: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + storage: {storageClassName: gp3, size: 500Gi} + splitBrainPolicy: + sitePriorities: [iad, pdx] +``` + ## Sizing and compatibility -- Minimum: 2 sites, both `primary-candidate`. -- Maximum: none imposed by the CRD. Practical limits derive from +- Minimum: 2 sites, both `primary-candidate`; readers are additional sites. +- Maximum: 16 sites. Practical limits derive from replication cost on the primary (each replica opens an I/O thread) and from DNS/LB churn at failover time. -- Every site is managed identically at the Kubernetes level — same - Deployment/Service/PVC shape, same role/permission model. Adding a - site is a CRD edit plus a reconcile. +- Every site receives a Deployment, PVC, client MySQL Service, internal + administrative Service, and per-site MySQL ConfigMap. Reader client + endpoints have stricter health gating than other site Services. ## Known limitations - `dr-only` sites cannot be auto-promoted. Region-level loss of every primary-candidate site requires manual intervention. -- The ordered rolling-update path walks a single active/standby pair - per update cycle; additional sites observe spec drift and get - rolled on subsequent cycles. -- Planned failover targets only `primary-candidate` sites. A `dr-only` - site must be deliberately reclassified before it can be promoted. +- Planned failover targets only `primary-candidate` sites. A `dr-only` or + `read-only` site must be deliberately reclassified, with `lbIP` and + `taintNodeSelector` supplied, before it can be promoted. diff --git a/docs/docs/operations.mdx b/docs/docs/operations.mdx index 0b3e9ef..7b2d1ff 100644 --- a/docs/docs/operations.mdx +++ b/docs/docs/operations.mdx @@ -141,7 +141,7 @@ If both sites report `writable` (split brain), the operator will **not** take au STOP REPLICA; RESET REPLICA ALL; CHANGE REPLICATION SOURCE TO - SOURCE_HOST='mysql-orders-iad.orders.svc.cluster.local', + SOURCE_HOST='mysql-orders-iad-internal.orders.svc.cluster.local', SOURCE_USER='replicator', SOURCE_PASSWORD='', SOURCE_AUTO_POSITION=1; @@ -193,6 +193,52 @@ If the prefix doesn't match the observed `divergentGtid` (wrong site, stale copy A `RecloneRequested` Normal Event marks the start. The site will temporarily appear unreachable during the MySQL restart that follows the clone. Progress is reported in the `Bootstrapping` condition. +### Recloning a read-only site + +A reader is a valid clone recipient but never a donor. If a reader has lost +its PVC or source convergence is blocked and its data can be discarded, use +the plugin's destructive cold-reclone confirmation: + +```bash +kubectl bloodraven reclone --cold +``` + +For example: + +```bash +kubectl bloodraven reclone orders reader --cold +``` + +The operator validates a uniquely writable `primary-candidate` donor before +starting `CLONE INSTANCE`. The reader client Service has no healthy endpoint +while clone, catch-up, or direct-source convergence is incomplete. Clone +reads from the active primary at full speed and is not throttled. + +### Troubleshooting blocked source convergence + +Inspect the generic source fields independently of old-primary recovery: + +```bash +kubectl get mysqlfailovergroup orders \ + -o jsonpath='{range .status.sites[*]}{.name}: source={.sourceHost} state={.sourceConvergenceState} reason={.sourceConvergenceReason}{"\n"}{end}' +``` + +- `Pending/SourceMismatch` means the observed source is not the confirmed + primary and convergence has not completed. +- `Pending/ProbeFailed` or `Pending/MutationFailed` means a probe or bounded + `STOP REPLICA` / `CHANGE REPLICATION SOURCE TO` / `START REPLICA` attempt + failed. Check the four stable convergence log events and their `stage`. +- `Blocked/GTIDDiverged` means the active primary does not contain the + follower's executed GTID set. Bloodraven intentionally leaves the channel + unchanged, or stopped if the second containment check failed after STOP. + Do not reset replication metadata to force a rejoin. Review the GTIDs and + preserve any required data before recloning. + +Convergence is also deliberately paused during bootstrap, ordered updates, +restore, topology freeze, pending promotion, planned failover, and split +brain. Resolve or allow that operation to finish before treating `Pending` +as a source fault. + ## PVC loss recovery runbook Use this when one site's MySQL data PVC is irrecoverable: cloud volume @@ -223,11 +269,10 @@ primary-candidate site. For normal old-primary divergence, use the GTID-prefix reclone flow above. For PVC loss, the target has no usable data to preserve, so the -operator accepts the cold reclone form: +operator accepts a cold reclone with an explicit destructive confirmation: ```bash -kubectl annotate mysqlfailovergroup orders \ - bloodraven.shipstream.io/reclone-site=iad +kubectl bloodraven reclone orders iad --cold ``` The operator emits `RecloneRequested` and drives the same clone path used diff --git a/docs/docs/planned-failover.mdx b/docs/docs/planned-failover.mdx index 35ff865..0542138 100644 --- a/docs/docs/planned-failover.mdx +++ b/docs/docs/planned-failover.mdx @@ -18,6 +18,11 @@ kubectl annotate mysqlfailovergroup orders \ The operator fences the current primary, waits for `pdx` to catch up on the fenced source's GTID set, promotes it, flips DNS, and clears the fence. When `spec.dragonfly.enabled=true`, the same request also waits for the target Dragonfly replica to catch up and promotes it before MySQL promotion. Status lands on `.status.plannedFailover.phase: Succeeded` with `transactionsLost: 0`. +Only `primary-candidate` sites are valid targets. `dr-only` and `read-only` +followers are never promoted, used for active DNS, or involved in node-taint +handoff. Readers remain direct followers and application read pools after the +switchover. + ## Why not `kubectl exec`? Manual promotion via three `kubectl exec ... SET GLOBAL ...` commands (see [Operations](./operations.mdx#manual-promotion-fallback-operator-unreachable)) works but has four real problems: @@ -133,7 +138,7 @@ Every path out of `Draining`, `WaitingForLag`, or `Promoting` either completes a | Failure | Observable | Rollback? | | --- | --- | --- | -| Target is unknown / dr-only / not replicating | `phase: Failed`, `reason: TargetUnhealthy` or `UnknownSite`, event `PlannedFailoverRejected` | no fence applied | +| Target is unknown / dr-only / read-only / not replicating | `phase: Failed`, `reason: TargetUnhealthy` or `UnknownSite`, event `PlannedFailoverRejected` | no fence applied | | Anti-flap cooldown active + `onCooldown: reject` | `phase: Failed`, `reason: CooldownActive`, message includes `retry after ...` | no fence applied | | Anti-flap cooldown active + `onCooldown: defer` | `phase: Deferred`, `reason: CooldownActive`, `retryAfter` populated; annotation retained | no fence applied; reconciler retries automatically | | Admin removes the annotation while `Deferred` | `phase: Failed`, `reason: Cancelled` | no fence applied | @@ -150,6 +155,28 @@ Every path out of `Draining`, `WaitingForLag`, or `Promoting` either completes a The topology-manager's automatic cross-site evaluation is **paused** while a planned failover is in flight (via the `plannedFailoverActive` guard). When the planned path stamps `Succeeded` or `Failed`, the guard is released and the automatic path resumes. +## Follower convergence after success + +Once the target is confirmed writable and planned-failover mutation is no +longer in flight, Bloodraven converges every other candidate, `dr-only`, and +`read-only` follower directly to the new primary. A chained or stale source is +not considered healthy merely because its replication threads are running. + +Before `STOP REPLICA`, the new primary's `GTID_EXECUTED` must contain the +follower's executed set. Bloodraven repeats this check after STOP to close the +SQL-applier race, then changes the source and starts replication without +resetting replication metadata. A failure is retried boundedly on subsequent +polls. GTID non-containment is reported per site as +`sourceConvergenceState: Blocked` and +`sourceConvergenceReason: GTIDDiverged`; the operator does not discard the +follower's extra transactions or silently repoint it. + +Reader client endpoints remain absent until MySQL is read-only, both threads +are healthy, direct-source convergence is `Converged`, and known lag is within +the effective reader threshold. A blocked or lagging reader does not change +the successful planned-failover result or the group's shared Ready/Degraded +conditions. + ## Anti-flap cooldown The planned path writes `status.lastFailover` on success, exactly like the emergency path. This means: diff --git a/docs/docs/playground.mdx b/docs/docs/playground.mdx index 9cd5eed..2b1e151 100644 --- a/docs/docs/playground.mdx +++ b/docs/docs/playground.mdx @@ -17,7 +17,7 @@ The playground is the recommended learning path before production. It lets you s ## What you get -- **Two-site MySQL cluster** (IAD + PDX) managed by the Bloodraven operator +- **Three-site MySQL cluster** with promotable IAD/PDX sites and a dedicated non-promotable reader - **Real-time dashboard** showing site health, replication state, DNS records, and an event log - **Counter app** that writes to MySQL through the primary service, proving state persists across failovers - **Simulated external-dns pipeline** — the operator creates `DNSEndpoint` CRs, external-dns watches them and pushes records to an in-memory webhook provider, and the dashboard displays the results @@ -31,7 +31,7 @@ You need three tools installed: - **kubectl** — for talking to your cluster - **helm** — for deploying the operator -And a local Kubernetes cluster with at least 2 worker nodes. We recommend [k3d](https://k3d.io) because it's fast and lightweight, but kind and minikube work too. +And a local Kubernetes cluster with at least 3 worker nodes. The third worker is dedicated to the `read-only` reader so storage-loss testing is deterministic. We recommend [k3d](https://k3d.io) because it's fast and lightweight, but kind and minikube work too. ### Create a cluster with k3d (recommended) @@ -39,11 +39,11 @@ And a local Kubernetes cluster with at least 2 worker nodes. We recommend [k3d]( # Install k3d if you haven't already curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash -# Create a cluster with 2 worker nodes -k3d cluster create bloodraven --agents 2 +# Create a cluster with 3 worker nodes +k3d cluster create bloodraven --agents 3 ``` -That's it. You now have a 3-node cluster (1 server + 2 agents) ready to go. +That's it. You now have a 4-node cluster (1 server + 3 agents) ready to go.
Using kind or minikube instead @@ -58,13 +58,14 @@ nodes: - role: control-plane - role: worker - role: worker + - role: worker EOF ``` **minikube:** ```bash -minikube start --nodes=2 --cpus=2 --memory=2048 --driver=docker +minikube start --nodes=4 --cpus=2 --memory=2048 --driver=docker ```
@@ -79,15 +80,17 @@ From the root of the repository: This single script handles everything: -1. Verifies your cluster has at least 2 nodes -2. Labels nodes to simulate two data center sites (IAD and PDX) +1. Verifies your cluster has at least 3 worker nodes +2. Labels dedicated workers for IAD, PDX, and the reader zone 3. Builds all container images locally 4. Loads images into your cluster (auto-detects k3d, kind, or minikube) 5. Installs the CRDs, namespace, RBAC, and operator via Helm -6. Creates a `MysqlFailoverGroup` with two MySQL sites +6. Creates a `MysqlFailoverGroup` with two primary candidates and one `read-only` reader 7. Seeds a DNS record so the external-dns pipeline works immediately 8. Deploys the dashboard and counter app +The default `reader` site intentionally omits `lbIP` and `taintNodeSelector`. It demonstrates reader-specific `mysqlConf`, a site-only NodePort Service override, and `replication.readOnlyMaxLagSeconds`; the priorities remain `[iad, pdx]`, so the reader cannot be promoted. + Setup takes about 2 minutes depending on your machine. ## Expected timings @@ -205,10 +208,13 @@ failure: ```bash make chaos-list # List every scenario with its hypothesis make chaos-run SCENARIO=06-self-fence-isolated-primary +make chaos-run SCENARIO=40-reader-data-loss-reclone make chaos-run-all-profile PROFILE=smoke # Core failover subset (~5 minutes) make chaos-run-all # The full suite ``` +Scenario 40 deterministically interprets reader node/storage loss as Deployment scale-down plus PVC replacement on the dedicated reader worker. It continuously proves reader loss does not unset group `Ready`, verifies the reader client endpoint is shed during clone/catch-up, and confirms direct-source replication and endpoint return after auto-clone. + This same suite is Bloodraven's E2E test bed in CI: the full suite runs nightly against a real Kubernetes cluster, and the smoke profile gates every release — a tag cannot publish images or charts unless core @@ -217,7 +223,7 @@ failover works on a live cluster. The current status is on the ## Dragonfly co-management -The playground `MysqlFailoverGroup` enables `spec.dragonfly`, so Bloodraven also creates one Dragonfly StatefulSet per site (`playground-dragonfly-iad`, `playground-dragonfly-pdx`), one Dragonfly PodDisruptionBudget per site, and a single app-facing `playground-dragonfly` Service whose endpoints follow the active site. The Service selector AND-gates `shipstream.io/dragonfly-role=master` AND `shipstream.io/dragonfly-traffic=enabled`: removing the traffic label sheds an endpoint atomically, which is how planned failover avoids a window where both the old and new master would match the selector during `REPLTAKEOVER`. Disable Dragonfly by removing the `spec.dragonfly` block. +The playground `MysqlFailoverGroup` enables `spec.dragonfly`, so Bloodraven also creates one Dragonfly StatefulSet per site (`playground-dragonfly-iad`, `playground-dragonfly-pdx`, and `playground-dragonfly-reader`), one Dragonfly PodDisruptionBudget per site, and a single app-facing `playground-dragonfly` Service whose endpoints follow the active site. The Service selector AND-gates `shipstream.io/dragonfly-role=master` AND `shipstream.io/dragonfly-traffic=enabled`: removing the traffic label sheds an endpoint atomically, which is how planned failover avoids a window where both the old and new master would match the selector during `REPLTAKEOVER`. Disable Dragonfly by removing the `spec.dragonfly` block. Bloodraven supports Dragonfly `v1.38.0+` for managed deployments. The playground pins `docker.dragonflydb.io/dragonflydb/dragonfly:v1.38.0`; avoid older tags because several replication, snapshot, and expiry bugs were fixed before this baseline. diff --git a/examples/minimal-failovergroup.yaml b/examples/minimal-failovergroup.yaml index b9fb2d6..33afcfe 100644 --- a/examples/minimal-failovergroup.yaml +++ b/examples/minimal-failovergroup.yaml @@ -12,6 +12,11 @@ spec: dns: hostname: orders.az.example.com ttl: 60 + replication: + maxLagSeconds: 300 + readOnlyMaxLagSeconds: 30 + serviceTemplate: + type: ClusterIP sites: - name: iad zone: us-east-1a @@ -31,3 +36,16 @@ spec: storage: storageClassName: fast-ssd size: 100Gi + - name: reader + role: read-only + zone: us-east-1b + mysqlConf: + innodb_buffer_pool_size: 4G + serviceTemplate: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + storage: + storageClassName: fast-ssd + size: 100Gi diff --git a/internal/controller/archiver_poller.go b/internal/controller/archiver_poller.go index 853de04..4038127 100644 --- a/internal/controller/archiver_poller.go +++ b/internal/controller/archiver_poller.go @@ -161,8 +161,7 @@ func (p *archiverPoller) clearMetrics() { func buildSidecarClients(fg *v1alpha1.MysqlFailoverGroup) []*internalmysql.SidecarClient { out := make([]*internalmysql.SidecarClient, len(fg.Spec.Sites)) for i, site := range fg.Spec.Sites { - url := fmt.Sprintf("http://mysql-%s-%s.%s.svc.cluster.local:%d", - fg.Name, site.Name, fg.Namespace, sidecarPort) + url := fmt.Sprintf("http://%s:%d", internalSiteServiceHost(fg.Name, site.Name, fg.Namespace), sidecarPort) out[i] = internalmysql.NewSidecarClient(url) } return out diff --git a/internal/controller/backup_job.go b/internal/controller/backup_job.go index 47fca26..a530cfd 100644 --- a/internal/controller/backup_job.go +++ b/internal/controller/backup_job.go @@ -596,8 +596,7 @@ func mergeSecurityContexts(userPod *corev1.PodSecurityContext, userContainer *co // backupMySQLHost returns the in-cluster hostname of a site's MySQL service. func backupMySQLHost(fg *v1alpha1.MysqlFailoverGroup, siteName string) string { - return fmt.Sprintf("mysql-%s-%s.%s.svc.cluster.local:%d", - fg.Name, siteName, fg.Namespace, mysqlPort) + return fmt.Sprintf("%s:%d", internalSiteServiceHost(fg.Name, siteName, fg.Namespace), mysqlPort) } // resolveBackupStorage returns (outputURL, extraEnv, volumes, mounts) for diff --git a/internal/controller/backup_reconciler.go b/internal/controller/backup_reconciler.go index e5c42ae..9bfedf7 100644 --- a/internal/controller/backup_reconciler.go +++ b/internal/controller/backup_reconciler.go @@ -156,7 +156,7 @@ func (r *MysqlBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) if fg.Spec.Backup != nil && fg.Spec.Backup.MaxLagSecondsForSource > 0 { maxLag = fg.Spec.Backup.MaxLagSecondsForSource } - sourceSite, reason, err := selectSourceSite(&fg.Status, backup.Spec.SourceSiteOverride, maxLag) + sourceSite, reason, err := selectSourceSite(&fg, backup.Spec.SourceSiteOverride, maxLag) if err != nil { return r.failBackup(ctx, &backup, "NoHealthySource", err.Error()) } @@ -1001,12 +1001,20 @@ func timeOrZero(t *metav1.Time) time.Time { // selectSourceSite picks the MySQL site to dump from. Replica first, // primary fallback. Returns the chosen site name and a short reason // string for events. `override` wins when set and the named site exists. -func selectSourceSite(status *v1alpha1.MysqlFailoverGroupStatus, override string, maxLagSeconds int64) (string, string, error) { - if status == nil || len(status.Sites) == 0 { +func selectSourceSite(fg *v1alpha1.MysqlFailoverGroup, override string, maxLagSeconds int64) (string, string, error) { + if fg == nil || len(fg.Status.Sites) == 0 { return "", "", fmt.Errorf("failover group has no observed site status yet") } + status := &fg.Status if override != "" { + specSite := fg.Spec.SiteByName(override) + if specSite == nil { + return "", "", fmt.Errorf("sourceSiteOverride %q does not match any configured site", override) + } + if specSite.IsReadOnlyReader() { + return "", "", fmt.Errorf("sourceSiteOverride %q names a read-only site, which cannot be a backup source", override) + } for i := range status.Sites { if status.Sites[i].Name == override { return override, "override", nil @@ -1019,33 +1027,34 @@ func selectSourceSite(status *v1alpha1.MysqlFailoverGroupStatus, override string return "", "", fmt.Errorf("failover group has no active site yet") } - var primary, replica *v1alpha1.SiteStatus + var primary *v1alpha1.SiteStatus + var replicas []*v1alpha1.SiteStatus for i := range status.Sites { s := &status.Sites[i] if s.Name == status.ActiveSite { primary = s continue } - if replica == nil { - replica = s + specSite := fg.Spec.SiteByName(s.Name) + if specSite != nil && !specSite.IsReadOnlyReader() { + replicas = append(replicas, s) } } - if replica != nil && replica.State == "read-only" && replica.Replicating { - if replica.SecondsBehindSource == nil { - return replica.Name, "replica-preferred", nil - } - if *replica.SecondsBehindSource <= maxLagSeconds { - return replica.Name, "replica-preferred", nil + for _, replica := range replicas { + if replica.State == "read-only" && replica.Replicating { + if replica.SecondsBehindSource == nil || *replica.SecondsBehindSource <= maxLagSeconds { + return replica.Name, "replica-preferred", nil + } } } - if primary != nil && primary.State == "writable" { + primarySpec := fg.Spec.SiteByName(status.ActiveSite) + if primary != nil && primary.State == "writable" && primarySpec != nil && primarySpec.IsPromotable() { return primary.Name, "primary-fallback", nil } - return "", "", fmt.Errorf("no healthy source (primary=%s, replica=%s)", - siteStateString(primary), siteStateString(replica)) + return "", "", fmt.Errorf("no healthy source (primary=%s)", siteStateString(primary)) } func siteStateString(s *v1alpha1.SiteStatus) string { diff --git a/internal/controller/backup_reconciler_test.go b/internal/controller/backup_reconciler_test.go index 0607eff..16427b5 100644 --- a/internal/controller/backup_reconciler_test.go +++ b/internal/controller/backup_reconciler_test.go @@ -21,6 +21,13 @@ import ( // --- selectSourceSite ---------------------------------------------------- +func backupTestGroup(status *v1alpha1.MysqlFailoverGroupStatus) *v1alpha1.MysqlFailoverGroup { + return &v1alpha1.MysqlFailoverGroup{ + Spec: v1alpha1.MysqlFailoverGroupSpec{Sites: []v1alpha1.SiteSpec{{Name: "iad"}, {Name: "pdx"}}}, + Status: *status, + } +} + func TestSelectSourceSite_ReplicaHealthy_PicksReplica(t *testing.T) { lag := int64(30) status := &v1alpha1.MysqlFailoverGroupStatus{ @@ -30,7 +37,7 @@ func TestSelectSourceSite_ReplicaHealthy_PicksReplica(t *testing.T) { {Name: "pdx", State: "read-only", Replicating: true, SecondsBehindSource: &lag}, }, } - site, reason, err := selectSourceSite(status, "", 300) + site, reason, err := selectSourceSite(backupTestGroup(status), "", 300) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -51,7 +58,7 @@ func TestSelectSourceSite_ReplicaLagging_FallsBackToPrimary(t *testing.T) { {Name: "pdx", State: "read-only", Replicating: true, SecondsBehindSource: &lag}, }, } - site, reason, err := selectSourceSite(status, "", 300) + site, reason, err := selectSourceSite(backupTestGroup(status), "", 300) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -71,7 +78,7 @@ func TestSelectSourceSite_ReplicaUnreachable_FallsBackToPrimary(t *testing.T) { {Name: "pdx", State: "unreachable"}, }, } - site, _, err := selectSourceSite(status, "", 300) + site, _, err := selectSourceSite(backupTestGroup(status), "", 300) if err != nil || site != "iad" { t.Fatalf("want iad, got site=%s err=%v", site, err) } @@ -85,7 +92,7 @@ func TestSelectSourceSite_NoHealthySource_Error(t *testing.T) { {Name: "pdx", State: "unreachable"}, }, } - if _, _, err := selectSourceSite(status, "", 300); err == nil { + if _, _, err := selectSourceSite(backupTestGroup(status), "", 300); err == nil { t.Fatal("expected error when both sites unhealthy") } } @@ -98,7 +105,7 @@ func TestSelectSourceSite_OverrideWins(t *testing.T) { {Name: "pdx", State: "unreachable"}, }, } - site, reason, err := selectSourceSite(status, "pdx", 300) + site, reason, err := selectSourceSite(backupTestGroup(status), "pdx", 300) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -115,11 +122,32 @@ func TestSelectSourceSite_OverrideUnknown_Error(t *testing.T) { {Name: "pdx", State: "read-only"}, }, } - if _, _, err := selectSourceSite(status, "nope", 300); err == nil { + if _, _, err := selectSourceSite(backupTestGroup(status), "nope", 300); err == nil { t.Fatal("expected error for unknown override site") } } +func TestSelectSourceSite_ExcludesReadOnlyRole(t *testing.T) { + lag := int64(0) + fg := &v1alpha1.MysqlFailoverGroup{ + Spec: v1alpha1.MysqlFailoverGroupSpec{Sites: []v1alpha1.SiteSpec{ + {Name: "iad"}, {Name: "pdx", Role: v1alpha1.SiteRoleDROnly}, {Name: "reader", Role: v1alpha1.SiteRoleReadOnly}, + }}, + Status: v1alpha1.MysqlFailoverGroupStatus{ActiveSite: "iad", Sites: []v1alpha1.SiteStatus{ + {Name: "iad", State: "writable"}, + {Name: "reader", State: "read-only", Replicating: true, SecondsBehindSource: &lag}, + {Name: "pdx", State: "read-only", Replicating: true, SecondsBehindSource: &lag}, + }}, + } + site, _, err := selectSourceSite(fg, "", 300) + if err != nil || site != "pdx" { + t.Fatalf("selected %q, err=%v", site, err) + } + if _, _, err := selectSourceSite(fg, "reader", 300); err == nil { + t.Fatal("explicit reader backup source must be rejected") + } +} + // --- BuildBackupJob ------------------------------------------------------ func fgWithBackup() *v1alpha1.MysqlFailoverGroup { @@ -212,7 +240,7 @@ func TestBuildBackupJob_S3_EnvAndConfig(t *testing.T) { for _, e := range c.Env { envMap[e.Name] = e.Value } - if h := envMap["BLOODRAVEN_MYSQL_HOST"]; !strings.Contains(h, "mysql-lion-pdx.ns.svc.cluster.local:3306") { + if h := envMap["BLOODRAVEN_MYSQL_HOST"]; !strings.Contains(h, "mysql-lion-pdx-internal.ns.svc.cluster.local:3306") { t.Errorf("bad host env: %q", h) } if envMap["BLOODRAVEN_S3_BUCKET"] != "bloodraven-backups" { diff --git a/internal/controller/credentials.go b/internal/controller/credentials.go index 6595b98..cf67806 100644 --- a/internal/controller/credentials.go +++ b/internal/controller/credentials.go @@ -60,10 +60,14 @@ func (r *MysqlFailoverGroupReconciler) reconcileCredentials(ctx context.Context, operatorPass := string(operatorSecret.Data["password"]) rootPass := string(operatorSecret.Data["MYSQL_ROOT_PASSWORD"]) - primaryHost := fmt.Sprintf("mysql-%s-primary.%s.svc.cluster.local:%d", fg.Name, fg.Namespace, mysqlPort) + activeSite := fg.Spec.SiteByName(fg.Status.ActiveSite) + if activeSite == nil || !activeSite.IsPromotable() { + return fmt.Errorf("active site %q is not a primary-candidate", fg.Status.ActiveSite) + } + primaryHost := fmt.Sprintf("%s:%d", internalSiteServiceHost(fg.Name, activeSite.Name, fg.Namespace), mysqlPort) tlsConfigName := "" if fg.Spec.TLS != nil { - tlsConfigName, err = mysqlTLSConfig(ctx, r.Client, fg, primaryServiceHost(fg.Name, fg.Namespace)) + tlsConfigName, err = mysqlTLSConfig(ctx, r.Client, fg, siteServiceHost(fg.Name, activeSite.Name, fg.Namespace)) if err != nil { return fmt.Errorf("configure TLS for credential reconciliation: %w", err) } diff --git a/internal/controller/mysql_tls.go b/internal/controller/mysql_tls.go index 08c65d8..191af45 100644 --- a/internal/controller/mysql_tls.go +++ b/internal/controller/mysql_tls.go @@ -77,6 +77,14 @@ func siteServiceHost(group, site, namespace string) string { return fmt.Sprintf("mysql-%s-%s.%s.svc.cluster.local", group, site, namespace) } +func internalSiteServiceName(group, site string) string { + return fmt.Sprintf("mysql-%s-%s-internal", group, site) +} + +func internalSiteServiceHost(group, site, namespace string) string { + return fmt.Sprintf("%s.%s.svc.cluster.local", internalSiteServiceName(group, site), namespace) +} + func primaryServiceHost(group, namespace string) string { return fmt.Sprintf("mysql-%s-primary.%s.svc.cluster.local", group, namespace) } diff --git a/internal/controller/mysql_tls_test.go b/internal/controller/mysql_tls_test.go index 5613207..5f26c19 100644 --- a/internal/controller/mysql_tls_test.go +++ b/internal/controller/mysql_tls_test.go @@ -28,6 +28,9 @@ func TestMySQLServiceHosts(t *testing.T) { if got, want := siteServiceHost("orders", "iad", "shop"), "mysql-orders-iad.shop.svc.cluster.local"; got != want { t.Fatalf("siteServiceHost() = %q, want %q", got, want) } + if got, want := internalSiteServiceHost("orders", "iad", "shop"), "mysql-orders-iad-internal.shop.svc.cluster.local"; got != want { + t.Fatalf("internalSiteServiceHost() = %q, want %q", got, want) + } if got, want := primaryServiceHost("orders", "shop"), "mysql-orders-primary.shop.svc.cluster.local"; got != want { t.Fatalf("primaryServiceHost() = %q, want %q", got, want) } diff --git a/internal/controller/reclone_test.go b/internal/controller/reclone_test.go index 9eb058e..3c6c58d 100644 --- a/internal/controller/reclone_test.go +++ b/internal/controller/reclone_test.go @@ -90,6 +90,14 @@ func TestValidateRecloneRequest_ColdReclone_WithConfirm_OK(t *testing.T) { } } +func TestValidateRecloneRequest_ReadOnlyRecipient_OK(t *testing.T) { + fg := recloneFG([]string{"iad", "pdx", "reader"}, nil) + fg.Spec.Sites[2].Role = v1alpha1.SiteRoleReadOnly + if err := validateRecloneRequest(fg, RecloneRequest{Site: "reader", GtidPrefix: "confirm=orders"}); err != nil { + t.Errorf("cold reader reclone with correct confirm token should succeed, got %v", err) + } +} + func TestValidateRecloneRequest_ColdReclone_WrongConfirm_Rejected(t *testing.T) { fg := recloneFG([]string{"iad", "pdx"}, nil) err := validateRecloneRequest(fg, RecloneRequest{Site: "iad", GtidPrefix: "confirm=wrongfg"}) diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go index 1a82027..7258cdf 100644 --- a/internal/controller/reconciler.go +++ b/internal/controller/reconciler.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "os" "sort" @@ -50,12 +51,14 @@ const ( labelManagedBy = "app.kubernetes.io/managed-by" managerName = "bloodraven" - specHashAnnotation = "shipstream.io/spec-hash" + specHashAnnotation = "shipstream.io/spec-hash" + managedServiceAnnotations = "shipstream.io/managed-service-annotations" + configMapRenderVersion = "site-config-v1" // Bump when the rendered MySQL Deployment pod spec changes without a // corresponding user-facing spec field change, so existing pods roll // forward to the new safe defaults. - deploymentPodRenderVersion = "deployment-pod-render-v2-stable-relay-log" + deploymentPodRenderVersion = "deployment-pod-render-v3-site-config-internal-service" // RecloneAnnotation is set by an admin to trigger a reclone of a // specific site from the current primary via CLONE INSTANCE. @@ -194,9 +197,10 @@ func (r *MysqlFailoverGroupReconciler) Reconcile(ctx context.Context, req ctrl.R return result, err } - // Reconcile ConfigMap - if err := r.reconcileConfigMap(ctx, &fg); err != nil { - return ctrl.Result{}, fmt.Errorf("reconcile configmap: %w", err) + // Desired per-site ConfigMaps are created before any Deployment reference + // changes. This ordering makes migration from the legacy shared map safe. + if err := r.reconcileSiteConfigMaps(ctx, &fg); err != nil { + return ctrl.Result{}, fmt.Errorf("reconcile site configmaps: %w", err) } // Reconcile init-users ConfigMap for MySQL user creation on first boot. @@ -215,14 +219,18 @@ func (r *MysqlFailoverGroupReconciler) Reconcile(ctx context.Context, req ctrl.R // simultaneous restarts of both sites (which causes a TOTAL LOSS window). orderedUpdateActive := fg.Status.UpdatePhase != "" - // When a topology manager is already running for this CR, defer Deployment - // updates to the ordered update path: the reconciler firing on a CR spec + // Defer every existing Deployment to the ordered update path: the reconciler firing on a CR spec // change must not restart both sites simultaneously. The runner's // checkSpecDrift compares the desired hash against the live Deployment // annotation, so leaving the existing Deployment untouched is what causes // drift to be observed and the ordered update to start. New Deployments - // (initial bootstrap) are always created here since there's no manager yet. - managerRunning := r.Runner != nil && r.Runner.HasManager(req.NamespacedName) + // (initial bootstrap) are always created here. This guard deliberately does + // not depend on runner wiring or manager registration: existing Deployments + // are never safe to patch from the bulk reconciliation loop. + deploymentReader := client.Reader(r.Client) + if r.APIReader != nil { + deploymentReader = r.APIReader + } // Reconcile per-site resources for i, site := range fg.Spec.Sites { @@ -233,17 +241,15 @@ func (r *MysqlFailoverGroupReconciler) Reconcile(ctx context.Context, req ctrl.R } if !orderedUpdateActive { deferDeployment := false - if managerRunning { - var existing appsv1.Deployment - deployNN := types.NamespacedName{ - Namespace: fg.Namespace, - Name: resourceName(fg.Name, site.Name), - } - if err := r.Get(ctx, deployNN, &existing); err == nil { - deferDeployment = true - } else if !errors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("get deployment %s: %w", site.Name, err) - } + var existing appsv1.Deployment + deployNN := types.NamespacedName{ + Namespace: fg.Namespace, + Name: resourceName(fg.Name, site.Name), + } + if err := deploymentReader.Get(ctx, deployNN, &existing); err == nil { + deferDeployment = true + } else if !errors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get deployment %s: %w", site.Name, err) } if !deferDeployment { if err := r.reconcileDeployment(ctx, &fg, site, serverID, image); err != nil { @@ -254,6 +260,12 @@ func (r *MysqlFailoverGroupReconciler) Reconcile(ctx context.Context, req ctrl.R if err := r.reconcileSiteService(ctx, &fg, site); err != nil { return ctrl.Result{}, fmt.Errorf("reconcile site service %s: %w", site.Name, err) } + if err := r.reconcileInternalSiteService(ctx, &fg, site); err != nil { + return ctrl.Result{}, fmt.Errorf("reconcile internal site service %s: %w", site.Name, err) + } + } + if err := r.cleanupLegacyConfigMap(ctx, &fg); err != nil { + return ctrl.Result{}, fmt.Errorf("cleanup legacy configmap: %w", err) } // Reconcile shared services @@ -469,7 +481,13 @@ func (r *MysqlFailoverGroupReconciler) handleDeletion(ctx context.Context, fg *v // Remove taints for all configured site selectors. if r.Tainter != nil { for _, site := range fg.Spec.Sites { + if site.IsReadOnlyReader() { + continue + } selector := taintNodeSelectorString(site.TaintNodeSelector) + if selector == "" { + continue + } if err := r.Tainter.SetTaint(ctx, selector, fg.Name, false); err != nil { logger.Error(err, "failed to remove taint during shutdown", "site", site.Name) // Continue cleanup despite taint removal failure @@ -583,10 +601,23 @@ func commonLabels(fgName, siteName string) map[string]string { } } -func (r *MysqlFailoverGroupReconciler) reconcileConfigMap(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup) error { +func siteConfigMapName(group, site string) string { + return fmt.Sprintf("mysql-%s-%s-config", group, site) +} + +func (r *MysqlFailoverGroupReconciler) reconcileSiteConfigMaps(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup) error { + for _, site := range fg.Spec.Sites { + if err := r.reconcileSiteConfigMap(ctx, fg, site); err != nil { + return err + } + } + return nil +} + +func (r *MysqlFailoverGroupReconciler) reconcileSiteConfigMap(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup, site v1alpha1.SiteSpec) error { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("mysql-%s-config", fg.Name), + Name: siteConfigMapName(fg.Name, site.Name), Namespace: fg.Namespace, }, } @@ -602,14 +633,27 @@ func (r *MysqlFailoverGroupReconciler) reconcileConfigMap(ctx context.Context, f labelManagedBy: managerName, } cm.Data = map[string]string{ - "bloodraven.cnf": generateMyCnf(fg), + "bloodraven.cnf": generateMyCnf(fg, site), } return nil }) return err } -func generateMyCnf(fg *v1alpha1.MysqlFailoverGroup) string { +func normalizedMySQLSettings(raw map[string]string) map[string]string { + keys := make([]string, 0, len(raw)) + for key := range raw { + keys = append(keys, key) + } + sort.Strings(keys) + out := make(map[string]string, len(raw)) + for _, key := range keys { + out[strings.ReplaceAll(key, "_", "-")] = raw[key] + } + return out +} + +func generateMyCnf(fg *v1alpha1.MysqlFailoverGroup, sites ...v1alpha1.SiteSpec) string { // Base config settings := map[string]string{ "gtid-mode": "ON", @@ -658,12 +702,23 @@ func generateMyCnf(fg *v1alpha1.MysqlFailoverGroup) string { settings["max-binlog-size"] = maxSize } - // Apply user overrides - for k, v := range fg.Spec.MysqlConf { - // MySQL treats dashes and underscores in option names as equivalent. - // Canonicalize them before merging so an underscore-style user key - // replaces a dash-style default instead of emitting both spellings. - settings[strings.ReplaceAll(k, "_", "-")] = v + for k, v := range normalizedMySQLSettings(fg.Spec.MysqlConf) { + settings[k] = v + } + if len(sites) > 0 { + for k, v := range normalizedMySQLSettings(sites[0].MysqlConf) { + settings[k] = v + } + } + + // These settings are operator-owned safety invariants and cannot be + // weakened by group or site overrides. + for k, v := range map[string]string{ + "gtid-mode": "ON", "enforce-gtid-consistency": "ON", + "log-bin": "/var/lib/mysql/mysql-bin", "log-replica-updates": "ON", + "skip-replica-start": "ON", "plugin-load-add": "mysql_clone.so", + } { + settings[k] = v } // Build sorted output for deterministic ConfigMap content @@ -797,18 +852,12 @@ func (r *MysqlFailoverGroupReconciler) reconcileDeployment(ctx context.Context, sidecarImage := fg.Spec.SidecarImage - configMapName := fmt.Sprintf("mysql-%s-config", fg.Name) + configMapName := siteConfigMapName(fg.Name, site.Name) // Build the list of peer sidecar addresses. The sidecar treats // "all peers unreachable" as one half of the self-fencing // quorum, so every non-self site needs to be listed here. - peerNames := fg.Spec.PeerSiteNames(site.Name) - peerAddrs := make([]string, len(peerNames)) - for i, name := range peerNames { - peerAddrs[i] = fmt.Sprintf("mysql-%s-%s.%s.svc.cluster.local:%d", - fg.Name, name, fg.Namespace, sidecarPort) - } - peerAddresses := strings.Join(peerAddrs, ",") + peerAddresses := strings.Join(sitePeerAddresses(fg, site.Name), ",") bloodravenAddress := defaultBloodravenAddress(fg) @@ -1244,29 +1293,45 @@ func (r *MysqlFailoverGroupReconciler) reconcileSiteService(ctx context.Context, return err } svc.Labels = commonLabels(fg.Name, site.Name) - applyServiceAnnotations(svc, fg.Spec.ServiceTemplate) - svc.Spec = corev1.ServiceSpec{ - Type: serviceType(fg.Spec.ServiceTemplate), - Selector: map[string]string{ - labelAppName: "mysql", - labelInstance: fg.Name, - labelSite: site.Name, - }, - Ports: []corev1.ServicePort{ - { - Name: "mysql", - Port: mysqlPort, - TargetPort: intstr.FromInt32(mysqlPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "sidecar", - Port: sidecarPort, - TargetPort: intstr.FromInt32(sidecarPort), - Protocol: corev1.ProtocolTCP, - }, + applyManagedServiceAnnotations(svc, effectiveSiteServiceAnnotations(fg.Spec.ServiceTemplate, site.ServiceTemplate)) + selector := map[string]string{ + labelAppName: "mysql", + labelInstance: fg.Name, + labelSite: site.Name, + } + if site.IsReadOnlyReader() { + selector[labelHealthy] = "yes" + } + mutateServiceSpec(svc, effectiveSiteServiceType(fg.Spec.ServiceTemplate, site.ServiceTemplate), effectiveSiteExternalTrafficPolicy(fg.Spec.ServiceTemplate, site.ServiceTemplate), selector, []corev1.ServicePort{ + { + Name: "mysql", + Port: mysqlPort, + TargetPort: intstr.FromInt32(mysqlPort), + Protocol: corev1.ProtocolTCP, + NodePort: siteServiceNodePort(site.ServiceTemplate), }, + }) + svc.Spec.PublishNotReadyAddresses = false + return nil + }) + return err +} + +func (r *MysqlFailoverGroupReconciler) reconcileInternalSiteService(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup, site v1alpha1.SiteSpec) error { + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: internalSiteServiceName(fg.Name, site.Name), Namespace: fg.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { + if err := controllerutil.SetControllerReference(fg, svc, r.Scheme); err != nil { + return err } + svc.Labels = commonLabels(fg.Name, site.Name) + applyManagedServiceAnnotations(svc, nil) + mutateServiceSpec(svc, corev1.ServiceTypeClusterIP, "", map[string]string{ + labelAppName: "mysql", labelInstance: fg.Name, labelSite: site.Name, + }, []corev1.ServicePort{ + {Name: "mysql", Port: mysqlPort, TargetPort: intstr.FromInt32(mysqlPort), Protocol: corev1.ProtocolTCP}, + {Name: "sidecar", Port: sidecarPort, TargetPort: intstr.FromInt32(sidecarPort), Protocol: corev1.ProtocolTCP}, + }) + svc.Spec.PublishNotReadyAddresses = true return nil }) return err @@ -1290,22 +1355,19 @@ func (r *MysqlFailoverGroupReconciler) reconcilePrimaryService(ctx context.Conte labelFailoverGroup: fg.Name, labelManagedBy: managerName, } - applyServiceAnnotations(svc, fg.Spec.ServiceTemplate) - svc.Spec = corev1.ServiceSpec{ - Type: serviceType(fg.Spec.ServiceTemplate), - Selector: map[string]string{ - labelInstance: fg.Name, - labelRole: "primary", - }, - Ports: []corev1.ServicePort{ - { - Name: "mysql", - Port: mysqlPort, - TargetPort: intstr.FromInt32(mysqlPort), - Protocol: corev1.ProtocolTCP, - }, + applyManagedServiceAnnotations(svc, serviceAnnotations(fg.Spec.ServiceTemplate)) + mutateServiceSpec(svc, serviceType(fg.Spec.ServiceTemplate), serviceExternalTrafficPolicy(fg.Spec.ServiceTemplate), map[string]string{ + labelInstance: fg.Name, + labelRole: "primary", + }, []corev1.ServicePort{ + { + Name: "mysql", + Port: mysqlPort, + TargetPort: intstr.FromInt32(mysqlPort), + Protocol: corev1.ProtocolTCP, }, - } + }) + svc.Spec.PublishNotReadyAddresses = false return nil }) return err @@ -1329,23 +1391,20 @@ func (r *MysqlFailoverGroupReconciler) reconcileReplicasService(ctx context.Cont labelFailoverGroup: fg.Name, labelManagedBy: managerName, } - applyServiceAnnotations(svc, fg.Spec.ServiceTemplate) - svc.Spec = corev1.ServiceSpec{ - Type: serviceType(fg.Spec.ServiceTemplate), - Selector: map[string]string{ - labelInstance: fg.Name, - labelRole: "replica", - labelHealthy: "yes", - }, - Ports: []corev1.ServicePort{ - { - Name: "mysql", - Port: mysqlPort, - TargetPort: intstr.FromInt32(mysqlPort), - Protocol: corev1.ProtocolTCP, - }, + applyManagedServiceAnnotations(svc, serviceAnnotations(fg.Spec.ServiceTemplate)) + mutateServiceSpec(svc, serviceType(fg.Spec.ServiceTemplate), serviceExternalTrafficPolicy(fg.Spec.ServiceTemplate), map[string]string{ + labelInstance: fg.Name, + labelRole: "replica", + labelHealthy: "yes", + }, []corev1.ServicePort{ + { + Name: "mysql", + Port: mysqlPort, + TargetPort: intstr.FromInt32(mysqlPort), + Protocol: corev1.ProtocolTCP, }, - } + }) + svc.Spec.PublishNotReadyAddresses = false return nil }) return err @@ -1393,33 +1452,159 @@ func serviceType(tmpl *v1alpha1.ServiceTemplate) corev1.ServiceType { } // applyServiceAnnotations merges user-supplied service annotations into the Service. +func serviceExternalTrafficPolicy(tmpl *v1alpha1.ServiceTemplate) corev1.ServiceExternalTrafficPolicyType { + if tmpl != nil { + return tmpl.ExternalTrafficPolicy + } + return "" +} + +func effectiveSiteServiceType(group *v1alpha1.ServiceTemplate, site *v1alpha1.SiteServiceTemplate) corev1.ServiceType { + if site != nil && site.Type != "" { + return site.Type + } + return serviceType(group) +} + +func effectiveSiteExternalTrafficPolicy(group *v1alpha1.ServiceTemplate, site *v1alpha1.SiteServiceTemplate) corev1.ServiceExternalTrafficPolicyType { + if site != nil && site.ExternalTrafficPolicy != "" { + return site.ExternalTrafficPolicy + } + return serviceExternalTrafficPolicy(group) +} + +func siteServiceNodePort(site *v1alpha1.SiteServiceTemplate) int32 { + if site != nil { + return site.NodePort + } + return 0 +} + +func serviceAnnotations(tmpl *v1alpha1.ServiceTemplate) map[string]string { + if tmpl == nil { + return nil + } + return tmpl.Annotations +} + +// applyServiceAnnotations is retained for Dragonfly Services, whose resource +// lifecycle is separate from the MySQL Service mutation contract. func applyServiceAnnotations(svc *corev1.Service, tmpl *v1alpha1.ServiceTemplate) { - if tmpl == nil || len(tmpl.Annotations) == 0 { + if tmpl == nil { return } if svc.Annotations == nil { - svc.Annotations = make(map[string]string, len(tmpl.Annotations)) + svc.Annotations = make(map[string]string) } for k, v := range tmpl.Annotations { svc.Annotations[k] = v } } +func effectiveSiteServiceAnnotations(group *v1alpha1.ServiceTemplate, site *v1alpha1.SiteServiceTemplate) map[string]string { + out := make(map[string]string) + if group != nil { + for k, v := range group.Annotations { + out[k] = v + } + } + if site != nil { + for k, v := range site.Annotations { + out[k] = v + } + } + return out +} + +func applyManagedServiceAnnotations(svc *corev1.Service, desired map[string]string) { + if svc.Annotations == nil { + svc.Annotations = make(map[string]string) + } + var previous []string + _ = json.Unmarshal([]byte(svc.Annotations[managedServiceAnnotations]), &previous) + for _, key := range previous { + delete(svc.Annotations, key) + } + keys := make([]string, 0, len(desired)) + for k, v := range desired { + svc.Annotations[k] = v + keys = append(keys, k) + } + sort.Strings(keys) + encoded, _ := json.Marshal(keys) + svc.Annotations[managedServiceAnnotations] = string(encoded) +} + +func mutateServiceSpec(svc *corev1.Service, serviceType corev1.ServiceType, etp corev1.ServiceExternalTrafficPolicyType, selector map[string]string, ports []corev1.ServicePort) { + oldPorts := make(map[string]corev1.ServicePort, len(svc.Spec.Ports)) + for _, port := range svc.Spec.Ports { + oldPorts[port.Name] = port + } + external := serviceType == corev1.ServiceTypeNodePort || serviceType == corev1.ServiceTypeLoadBalancer + if external && etp == "" { + etp = corev1.ServiceExternalTrafficPolicyCluster + } + for i := range ports { + if external && ports[i].NodePort == 0 { + ports[i].NodePort = oldPorts[ports[i].Name].NodePort + } + if !external { + ports[i].NodePort = 0 + } + } + svc.Spec.Type = serviceType + svc.Spec.Selector = selector + svc.Spec.Ports = ports + if external { + svc.Spec.ExternalTrafficPolicy = etp + } else { + svc.Spec.ExternalTrafficPolicy = "" + } + if serviceType != corev1.ServiceTypeLoadBalancer { + svc.Spec.LoadBalancerIP = "" + svc.Spec.LoadBalancerSourceRanges = nil + svc.Spec.LoadBalancerClass = nil + svc.Spec.AllocateLoadBalancerNodePorts = nil + } + if serviceType != corev1.ServiceTypeLoadBalancer || etp != corev1.ServiceExternalTrafficPolicyLocal { + svc.Spec.HealthCheckNodePort = 0 + } +} + // syncPodLabels updates pod labels based on the CR status. // It updates replicas first, then primary, to prevent dual-primary in Service endpoints. func (r *MysqlFailoverGroupReconciler) syncPodLabels(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup) error { logger := log.FromContext(ctx) - if fg.Status.ActiveSite == "" { - return nil + statusByName := make(map[string]v1alpha1.SiteStatus, len(fg.Status.Sites)) + for _, siteStatus := range fg.Status.Sites { + statusByName[siteStatus.Name] = siteStatus } - - // Guard: status may not be populated yet - if len(fg.Status.Sites) < len(fg.Spec.Sites) { - return nil + statusComplete := len(statusByName) >= len(fg.Spec.Sites) + writableSites := 0 + for _, site := range fg.Spec.Sites { + status, found := statusByName[site.Name] + if !found { + statusComplete = false + continue + } + if status.State == "writable" { + writableSites++ + } + } + authorityValid := false + if statusComplete && fg.Status.ActiveSite != "" && writableSites == 1 { + for _, site := range fg.Spec.Sites { + if site.Name == fg.Status.ActiveSite && site.IsPromotable() && statusByName[site.Name].State == "writable" { + authorityValid = true + break + } + } } - // Determine which site is primary, which is replica + // Determine which site is primary, which is replica. Invalid or incomplete + // authority deliberately leaves every site non-primary and every reader + // non-serving, shedding stale Service endpoints rather than returning early. type siteInfo struct { spec v1alpha1.SiteSpec status v1alpha1.SiteStatus @@ -1428,12 +1613,13 @@ func (r *MysqlFailoverGroupReconciler) syncPodLabels(ctx context.Context, fg *v1 sites := make([]siteInfo, len(fg.Spec.Sites)) for i := range fg.Spec.Sites { + siteStatus := statusByName[fg.Spec.Sites[i].Name] sites[i] = siteInfo{ spec: fg.Spec.Sites[i], - status: fg.Status.Sites[i], + status: siteStatus, role: "replica", } - if fg.Status.ActiveSite == fg.Spec.Sites[i].Name { + if authorityValid && fg.Status.ActiveSite == fg.Spec.Sites[i].Name { sites[i].role = "primary" } } @@ -1491,7 +1677,14 @@ func (r *MysqlFailoverGroupReconciler) syncPodLabels(ctx context.Context, fg *v1 } healthy := "no" - if si.status.State == "writable" || si.status.State == "read-only" { + if si.spec.IsReadOnlyReader() { + if authorityValid && si.status.State == "read-only" && si.status.SourceConvergenceState == v1alpha1.SourceConvergenceConverged && + si.status.Replicating && si.status.SecondsBehindSource != nil && + canonicalSourceHost(si.status.SourceHost) == canonicalSourceHost(internalSiteServiceHost(fg.Name, fg.Status.ActiveSite, fg.Namespace)) && + *si.status.SecondsBehindSource <= fg.Spec.EffectiveReadOnlyMaxLagSeconds() { + healthy = "yes" + } + } else if si.status.State == "writable" || si.status.State == "read-only" { healthy = "yes" } @@ -1558,15 +1751,10 @@ func ComputeSpecHash(fg *v1alpha1.MysqlFailoverGroup, site v1alpha1.SiteSpec, tl // already relies on for resources/sidecarResources. fmt.Fprintf(h, "podSC=%v\n", fg.Spec.PodSecurityContext) fmt.Fprintf(h, "containerSC=%v\n", fg.Spec.ContainerSecurityContext) - // Sort mysqlConf keys for deterministic hash - keys := make([]string, 0, len(fg.Spec.MysqlConf)) - for k := range fg.Spec.MysqlConf { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - fmt.Fprintf(h, "mysql.%s=%s\n", k, fg.Spec.MysqlConf[k]) - } + fmt.Fprintf(h, "configMapRenderVersion=%s\n", configMapRenderVersion) + fmt.Fprintf(h, "configMapName=%s\n", siteConfigMapName(fg.Name, site.Name)) + fmt.Fprintf(h, "effectiveMyCnf=%s\n", generateMyCnf(fg, site)) + fmt.Fprintf(h, "peerAddresses=%s\n", strings.Join(sitePeerAddresses(fg, site.Name), ",")) // PITR settings affect both my.cnf (max_binlog_size) and the // sidecar env (archiver config). Either change must roll the pod. // Hash the EFFECTIVE values (with defaults filled in) rather than @@ -1641,6 +1829,15 @@ func ComputeSpecHash(fg *v1alpha1.MysqlFailoverGroup, site v1alpha1.SiteSpec, tl return hex.EncodeToString(h.Sum(nil))[:16] } +func sitePeerAddresses(fg *v1alpha1.MysqlFailoverGroup, siteName string) []string { + peerNames := fg.Spec.PeerSiteNames(siteName) + peerAddrs := make([]string, len(peerNames)) + for i, name := range peerNames { + peerAddrs[i] = fmt.Sprintf("%s:%d", internalSiteServiceHost(fg.Name, name, fg.Namespace), sidecarPort) + } + return peerAddrs +} + func int32Ptr(i int32) *int32 { return &i } // CRConfigToTopologyConfig extracts topology manager configuration from a CR. @@ -1679,21 +1876,82 @@ func CRConfigToTopologyConfig(fg *v1alpha1.MysqlFailoverGroup) TopologyConfig { LBIP: s.LBIP, Role: state.SiteRole(s.EffectiveRole()), TaintSelector: taintNodeSelectorString(s.TaintNodeSelector), - Host: fmt.Sprintf("mysql-%s-%s.%s.svc.cluster.local", fg.Name, s.Name, fg.Namespace), + Host: internalSiteServiceHost(fg.Name, s.Name, fg.Namespace), } } return TopologyConfig{ - Namespace: fg.Namespace, - Name: fg.Name, - Sites: sites, - PollInterval: pollInterval, - FailureThreshold: int(failureThreshold), - RecoveryThreshold: int(recoveryThreshold), - FailoverCooldown: failoverCooldown, - SitePriorities: sitePriorities, - DragonflyEnabled: dragonflyEnabled(fg), + Namespace: fg.Namespace, + Name: fg.Name, + Sites: sites, + PollInterval: pollInterval, + FailureThreshold: int(failureThreshold), + RecoveryThreshold: int(recoveryThreshold), + FailoverCooldown: failoverCooldown, + MaxLagSeconds: fg.Spec.EffectiveMaxLagSeconds(), + ReadOnlyMaxLagSeconds: fg.Spec.EffectiveReadOnlyMaxLagSeconds(), + SitePriorities: sitePriorities, + DragonflyEnabled: dragonflyEnabled(fg), + } +} + +func (r *MysqlFailoverGroupReconciler) cleanupLegacyConfigMap(ctx context.Context, fg *v1alpha1.MysqlFailoverGroup) error { + reader := client.Reader(r.Client) + if r.APIReader != nil { + reader = r.APIReader + } + legacy := fmt.Sprintf("mysql-%s-config", fg.Name) + desiredConfigs := make(map[string]string, len(fg.Spec.Sites)) + for _, site := range fg.Spec.Sites { + desiredConfig := siteConfigMapName(fg.Name, site.Name) + var cm corev1.ConfigMap + if err := reader.Get(ctx, types.NamespacedName{Namespace: fg.Namespace, Name: desiredConfig}, &cm); err != nil { + return nil + } + desiredConfigs[resourceName(fg.Name, site.Name)] = desiredConfig } + + var deployments appsv1.DeploymentList + if err := reader.List(ctx, &deployments, + client.InNamespace(fg.Namespace), + client.MatchingLabels{labelAppName: "mysql", labelInstance: fg.Name}, + ); err != nil { + return nil + } + seenDesired := make(map[string]bool, len(desiredConfigs)) + for i := range deployments.Items { + deployment := &deployments.Items[i] + configName := deploymentConfigMapName(deployment) + if expected, desired := desiredConfigs[deployment.Name]; desired { + seenDesired[deployment.Name] = true + if configName != expected { + return nil + } + continue + } + if configName == legacy { + return nil + } + } + for deploymentName := range desiredConfigs { + if !seenDesired[deploymentName] { + return nil + } + } + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: legacy, Namespace: fg.Namespace}} + if err := r.Delete(ctx, cm); err != nil && !errors.IsNotFound(err) { + return err + } + return nil +} + +func deploymentConfigMapName(deployment *appsv1.Deployment) string { + for _, volume := range deployment.Spec.Template.Spec.Volumes { + if volume.Name == "config" && volume.ConfigMap != nil { + return volume.ConfigMap.Name + } + } + return "" } func taintNodeSelectorString(selector map[string]string) string { @@ -1719,6 +1977,9 @@ func (r *MysqlFailoverGroupReconciler) ReconcileSiteDeployment(ctx context.Conte for i, site := range fg.Spec.Sites { if site.Name == siteName { siteFound = true + if err := r.reconcileSiteConfigMap(ctx, &fg, site); err != nil { + return fmt.Errorf("reconcile site configmap: %w", err) + } serverID := int32(101 + i) if err := r.reconcileDeployment(ctx, &fg, site, serverID, image); err != nil { return err diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.go index d60b9e8..2b3dca1 100644 --- a/internal/controller/reconciler_test.go +++ b/internal/controller/reconciler_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "io" "log/slog" "strings" @@ -129,7 +130,7 @@ func TestReconcile_CreatesConfigMap(t *testing.T) { var cm corev1.ConfigMap if err := c.Get(context.Background(), types.NamespacedName{ - Name: "mysql-lion-config", Namespace: "shared-lion", + Name: "mysql-lion-dc1-config", Namespace: "shared-lion", }, &cm); err != nil { t.Fatalf("configmap not created: %v", err) } @@ -363,7 +364,7 @@ func TestReconcile_TLSConfig(t *testing.T) { var cm corev1.ConfigMap if err := c.Get(context.Background(), types.NamespacedName{ - Name: "mysql-lion-config", Namespace: "shared-lion", + Name: "mysql-lion-dc1-config", Namespace: "shared-lion", }, &cm); err != nil { t.Fatalf("configmap not found: %v", err) } @@ -394,7 +395,7 @@ func TestReconcile_MysqlConfOverrides(t *testing.T) { var cm corev1.ConfigMap if err := c.Get(context.Background(), types.NamespacedName{ - Name: "mysql-lion-config", Namespace: "shared-lion", + Name: "mysql-lion-dc1-config", Namespace: "shared-lion", }, &cm); err != nil { t.Fatalf("configmap not found: %v", err) } @@ -791,11 +792,9 @@ func TestReconcile_TLSSecretHashTriggersRollout(t *testing.T) { t.Fatalf("update TLS secret: %v", err) } - // Reconcile again — hash should change. - if _, err := r.Reconcile(context.Background(), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: "lion", Namespace: "shared-lion"}, - }); err != nil { - t.Fatalf("reconcile after cert rotation failed: %v", err) + // Existing Deployments are updated only through the ordered site path. + if err := r.reconcileDeployment(context.Background(), fg, fg.Spec.Sites[0], 101, fg.Spec.Image); err != nil { + t.Fatalf("ordered site reconcile after cert rotation failed: %v", err) } if err := c.Get(context.Background(), types.NamespacedName{ @@ -825,6 +824,319 @@ func TestComputeSpecHash_StableWithoutTLS(t *testing.T) { } } +func TestComputeSpecHash_IncludesAllPeerAddresses(t *testing.T) { + fg := newTestFG() + site := fg.Spec.Sites[0] + twoSiteHash := ComputeSpecHash(fg, site, nil, nil) + + reader := v1alpha1.SiteSpec{Name: "reader", Role: v1alpha1.SiteRoleReadOnly} + fg.Spec.Sites = append(fg.Spec.Sites, reader) + threeSiteHash := ComputeSpecHash(fg, site, nil, nil) + if threeSiteHash == twoSiteHash { + t.Fatal("adding a reader did not change an existing site's pod hash") + } + if got, want := strings.Join(sitePeerAddresses(fg, "dc1"), ","), + "mysql-lion-dc2-internal.shared-lion.svc.cluster.local:8080,mysql-lion-reader-internal.shared-lion.svc.cluster.local:8080"; got != want { + t.Fatalf("peer addresses = %q, want %q", got, want) + } + r, c := newReconciler(fg) + if err := r.reconcileDeployment(context.Background(), fg, site, 101, fg.Spec.Image); err != nil { + t.Fatal(err) + } + var deployment appsv1.Deployment + if err := c.Get(context.Background(), types.NamespacedName{Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace}, &deployment); err != nil { + t.Fatal(err) + } + var renderedPeers string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name != "sidecar" { + continue + } + for _, env := range container.Env { + if env.Name == "PEER_ADDRESSES" { + renderedPeers = env.Value + } + } + } + if renderedPeers != strings.Join(sitePeerAddresses(fg, site.Name), ",") { + t.Fatalf("rendered PEER_ADDRESSES = %q", renderedPeers) + } + + fg.Spec.Sites = fg.Spec.Sites[:2] + if got := ComputeSpecHash(fg, site, nil, nil); got != twoSiteHash { + t.Fatalf("removing reader did not restore deterministic surviving-site hash: got %s want %s", got, twoSiteHash) + } +} + +func TestReaderServicesAndSiteOverrides(t *testing.T) { + fg := newTestFG() + reader := v1alpha1.SiteSpec{ + Name: "reader", Role: v1alpha1.SiteRoleReadOnly, Zone: "reader-zone", + Storage: v1alpha1.StorageSpec{StorageClassName: "local", Size: resource.MustParse("10Gi")}, + ServiceTemplate: &v1alpha1.SiteServiceTemplate{ + Type: corev1.ServiceTypeNodePort, ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyLocal, + NodePort: 31001, Annotations: map[string]string{"site": "reader"}, + }, + } + fg.Spec.ServiceTemplate = &v1alpha1.ServiceTemplate{Annotations: map[string]string{"group": "yes", "site": "group"}} + r, c := newReconciler(fg) + if err := r.reconcileSiteService(context.Background(), fg, reader); err != nil { + t.Fatal(err) + } + if err := r.reconcileInternalSiteService(context.Background(), fg, reader); err != nil { + t.Fatal(err) + } + var external corev1.Service + if err := c.Get(context.Background(), types.NamespacedName{Name: "mysql-lion-reader", Namespace: fg.Namespace}, &external); err != nil { + t.Fatal(err) + } + if external.Spec.Selector[labelHealthy] != "yes" || len(external.Spec.Ports) != 1 || external.Spec.Ports[0].Name != "mysql" { + t.Fatalf("reader service = %#v", external.Spec) + } + if external.Spec.Ports[0].NodePort != 31001 || external.Spec.ExternalTrafficPolicy != corev1.ServiceExternalTrafficPolicyLocal { + t.Fatalf("reader exposure not applied: %#v", external.Spec) + } + if external.Annotations["group"] != "yes" || external.Annotations["site"] != "reader" { + t.Fatalf("annotations = %v", external.Annotations) + } + var internal corev1.Service + if err := c.Get(context.Background(), types.NamespacedName{Name: "mysql-lion-reader-internal", Namespace: fg.Namespace}, &internal); err != nil { + t.Fatal(err) + } + if internal.Spec.Type != corev1.ServiceTypeClusterIP || !internal.Spec.PublishNotReadyAddresses || internal.Spec.Selector[labelHealthy] != "" || len(internal.Spec.Ports) != 2 { + t.Fatalf("internal service = %#v", internal.Spec) + } +} + +func TestMutateServiceSpecLifecycle(t *testing.T) { + svc := &corev1.Service{Spec: corev1.ServiceSpec{ + ClusterIP: "10.0.0.10", ClusterIPs: []string{"10.0.0.10"}, Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{{Name: "mysql", Port: mysqlPort, NodePort: 31000}}, + HealthCheckNodePort: 32000, ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyLocal, + }} + port := []corev1.ServicePort{{Name: "mysql", Port: mysqlPort}} + mutateServiceSpec(svc, corev1.ServiceTypeLoadBalancer, corev1.ServiceExternalTrafficPolicyLocal, nil, port) + if svc.Spec.Ports[0].NodePort != 31000 || svc.Spec.ClusterIP != "10.0.0.10" { + t.Fatalf("allocated fields not preserved: %#v", svc.Spec) + } + port[0].NodePort = 31001 + mutateServiceSpec(svc, corev1.ServiceTypeNodePort, corev1.ServiceExternalTrafficPolicyCluster, nil, port) + if svc.Spec.Ports[0].NodePort != 31001 { + t.Fatalf("explicit nodePort not applied: %#v", svc.Spec.Ports) + } + mutateServiceSpec(svc, corev1.ServiceTypeClusterIP, "", nil, port) + if svc.Spec.Ports[0].NodePort != 0 || svc.Spec.ExternalTrafficPolicy != "" || svc.Spec.HealthCheckNodePort != 0 || svc.Spec.ClusterIP != "10.0.0.10" { + t.Fatalf("external to ClusterIP transition = %#v", svc.Spec) + } +} + +func TestGenerateMyCnfSitePrecedenceAndProtectedSettings(t *testing.T) { + fg := newTestFG() + fg.Spec.MysqlConf = map[string]string{"max_connections": "600", "sync-binlog": "0", "gtid_mode": "OFF"} + site := fg.Spec.Sites[0] + site.MysqlConf = map[string]string{"max-connections": "700", "log_bin": "/unsafe"} + cnf := generateMyCnf(fg, site) + for _, want := range []string{"max-connections=700", "sync-binlog=0", "gtid-mode=ON", "log-bin=/var/lib/mysql/mysql-bin"} { + if !strings.Contains(cnf, want) { + t.Fatalf("my.cnf missing %q:\n%s", want, cnf) + } + } + fg2 := fg.DeepCopy() + fg2.Spec.MysqlConf = map[string]string{"max-connections": "600", "sync_binlog": "0", "gtid-mode": "OFF"} + if ComputeSpecHash(fg, site, nil, nil) != ComputeSpecHash(fg2, site, nil, nil) { + t.Fatal("normalized equivalent MySQL settings must hash identically") + } +} + +func TestLegacyConfigMapCleanupIsReferenceDriven(t *testing.T) { + fg := newTestFG() + legacyName := "mysql-lion-config" + legacy := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: legacyName, Namespace: fg.Namespace}} + deployments := make([]client.Object, 0, len(fg.Spec.Sites)) + for _, site := range fg.Spec.Sites { + deployments = append(deployments, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace, Labels: commonLabels(fg.Name, site.Name)}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{ + Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: legacyName}}}, + }}}}}, + }) + } + objects := []client.Object{fg, legacy} + objects = append(objects, deployments...) + r, c := newReconciler(objects...) + if err := r.reconcileSiteConfigMaps(context.Background(), fg); err != nil { + t.Fatal(err) + } + if err := r.cleanupLegacyConfigMap(context.Background(), fg); err != nil { + t.Fatal(err) + } + var retained corev1.ConfigMap + if err := c.Get(context.Background(), types.NamespacedName{Name: legacyName, Namespace: fg.Namespace}, &retained); err != nil { + t.Fatalf("referenced legacy map was deleted: %v", err) + } + for _, site := range fg.Spec.Sites { + var deployment appsv1.Deployment + nn := types.NamespacedName{Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace} + if err := c.Get(context.Background(), nn, &deployment); err != nil { + t.Fatal(err) + } + deployment.Spec.Template.Spec.Volumes[0].ConfigMap.Name = siteConfigMapName(fg.Name, site.Name) + if err := c.Update(context.Background(), &deployment); err != nil { + t.Fatal(err) + } + } + if err := r.cleanupLegacyConfigMap(context.Background(), fg); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), types.NamespacedName{Name: legacyName, Namespace: fg.Namespace}, &retained); err == nil { + t.Fatal("unreferenced legacy map was retained") + } + if err := r.cleanupLegacyConfigMap(context.Background(), fg); err != nil { + t.Fatalf("idempotent cleanup after deletion failed: %v", err) + } +} + +func TestLegacyConfigMapCleanupRetentionMatrix(t *testing.T) { + for _, tc := range []struct { + name string + kind string + }{ + {name: "missing desired configmap", kind: "missing-config"}, + {name: "missing desired deployment", kind: "missing-deployment"}, + {name: "desired deployment has unexpected reference", kind: "unexpected-reference"}, + {name: "removed site deployment still references legacy", kind: "extra-legacy"}, + {name: "uncached deployment list fails", kind: "list-error"}, + } { + t.Run(tc.name, func(t *testing.T) { + fg := newTestFG() + legacyName := "mysql-lion-config" + objects := []client.Object{ + fg, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: legacyName, Namespace: fg.Namespace}}, + } + for i, site := range fg.Spec.Sites { + if tc.kind != "missing-config" || i != 0 { + objects = append(objects, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: siteConfigMapName(fg.Name, site.Name), Namespace: fg.Namespace}}) + } + if tc.kind == "missing-deployment" && i == 0 { + continue + } + configName := siteConfigMapName(fg.Name, site.Name) + if tc.kind == "unexpected-reference" && i == 0 { + configName = "mysql-lion-wrong-config" + } + objects = append(objects, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace, Labels: commonLabels(fg.Name, site.Name)}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{ + Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: configName}}}, + }}}}}, + }) + } + if tc.kind == "extra-legacy" { + objects = append(objects, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName(fg.Name, "removed"), Namespace: fg.Namespace, Labels: commonLabels(fg.Name, "removed")}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{ + Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: legacyName}}}, + }}}}}, + }) + } + r, c := newReconciler(objects...) + r.APIReader = c + if tc.kind == "list-error" { + r.APIReader = failingListReader{Reader: c} + } + if err := r.cleanupLegacyConfigMap(context.Background(), fg); err != nil { + t.Fatal(err) + } + var legacy corev1.ConfigMap + if err := c.Get(context.Background(), types.NamespacedName{Name: legacyName, Namespace: fg.Namespace}, &legacy); err != nil { + t.Fatalf("legacy ConfigMap was deleted without complete reference proof: %v", err) + } + }) + } +} + +func TestSyncPodLabelsFailsSafeAndRecovers(t *testing.T) { + lag0, lagHigh := int64(0), int64(301) + base := func() *v1alpha1.MysqlFailoverGroup { + fg := newTestFG() + fg.Spec.Sites = append(fg.Spec.Sites, v1alpha1.SiteSpec{Name: "reader", Role: v1alpha1.SiteRoleReadOnly}) + return fg + } + healthyStatuses := func() []v1alpha1.SiteStatus { + return []v1alpha1.SiteStatus{ + {Name: "dc1", State: "writable"}, + {Name: "dc2", State: "read-only", Replicating: true}, + {Name: "reader", State: "read-only", Replicating: true, SourceHost: "mysql-lion-dc1-internal.shared-lion.svc.cluster.local", SourceConvergenceState: v1alpha1.SourceConvergenceConverged, SecondsBehindSource: &lag0}, + } + } + for _, tc := range []struct { + name string + configure func(*v1alpha1.MysqlFailoverGroup) + wantPrimary bool + wantReaderYes bool + }{ + {name: "empty active", configure: func(fg *v1alpha1.MysqlFailoverGroup) { fg.Status.Sites = healthyStatuses() }}, + {name: "non-promotable active", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "reader" + fg.Status.Sites = healthyStatuses() + fg.Status.Sites[0].State = "read-only" + fg.Status.Sites[2].State = "writable" + }}, + {name: "incomplete status", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "dc1" + fg.Status.Sites = healthyStatuses()[:2] + }}, + {name: "source mismatch", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "dc1" + fg.Status.Sites = healthyStatuses() + fg.Status.Sites[2].SourceHost = "wrong" + }, wantPrimary: true}, + {name: "thread failure", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "dc1" + fg.Status.Sites = healthyStatuses() + fg.Status.Sites[2].Replicating = false + }, wantPrimary: true}, + {name: "lag failure", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "dc1" + fg.Status.Sites = healthyStatuses() + fg.Status.Sites[2].SecondsBehindSource = &lagHigh + }, wantPrimary: true}, + {name: "healthy recovery", configure: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Status.ActiveSite = "dc1" + fg.Status.Sites = healthyStatuses() + }, wantPrimary: true, wantReaderYes: true}, + } { + t.Run(tc.name, func(t *testing.T) { + fg := base() + tc.configure(fg) + primaryPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "dc1-pod", Namespace: fg.Namespace, Labels: map[string]string{ + labelAppName: "mysql", labelInstance: fg.Name, labelSite: "dc1", labelRole: "primary", labelHealthy: "yes", + }}} + readerPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "reader-pod", Namespace: fg.Namespace, Labels: map[string]string{ + labelAppName: "mysql", labelInstance: fg.Name, labelSite: "reader", labelRole: "replica", labelHealthy: "yes", + }}} + r, c := newReconciler(fg, primaryPod, readerPod) + if err := r.syncPodLabels(context.Background(), fg); err != nil { + t.Fatal(err) + } + var gotPrimary, gotReader corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Name: primaryPod.Name, Namespace: fg.Namespace}, &gotPrimary); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), types.NamespacedName{Name: readerPod.Name, Namespace: fg.Namespace}, &gotReader); err != nil { + t.Fatal(err) + } + if (gotPrimary.Labels[labelRole] == "primary") != tc.wantPrimary { + t.Fatalf("primary role label = %q, wantPrimary=%t", gotPrimary.Labels[labelRole], tc.wantPrimary) + } + if (gotReader.Labels[labelHealthy] == "yes") != tc.wantReaderYes { + t.Fatalf("reader healthy label = %q, wantHealthy=%t", gotReader.Labels[labelHealthy], tc.wantReaderYes) + } + }) + } +} + func TestComputeSpecHash_IncludesCredentialData(t *testing.T) { fg := newTestFG() site := fg.Spec.Sites[0] @@ -1119,7 +1431,7 @@ func TestBuildSiteDSNFromCreds(t *testing.T) { if !strings.Contains(dsn, "myuser:mypass@") { t.Errorf("DSN should contain credentials: %s", dsn) } - if !strings.Contains(dsn, "mysql-lion-dc1.shared-lion.svc.cluster.local") { + if !strings.Contains(dsn, "mysql-lion-dc1-internal.shared-lion.svc.cluster.local") { t.Errorf("DSN should contain site host: %s", dsn) } dsn = buildSiteDSNFromCreds("myuser", "mypass", fg, fg.Spec.Sites[0], "bloodraven-test-tls") @@ -1224,11 +1536,9 @@ func TestReconcile_DefersDeploymentUpdateWhenManagerRunning(t *testing.T) { } } -// TestReconcile_AppliesDeploymentUpdateWhenNoManager verifies that when no -// topology manager is running (initial deploy or pre-leader-election state), -// the reconciler still applies Deployment updates directly. This is the -// bootstrap path that must keep working even after the deferred-update change. -func TestReconcile_AppliesDeploymentUpdateWhenNoManager(t *testing.T) { +// TestReconcile_DefersExistingDeploymentWhenRunnerUnavailable verifies the +// fail-safe behavior does not depend on manager wiring. +func TestReconcile_DefersExistingDeploymentWhenRunnerUnavailable(t *testing.T) { ctx := context.Background() fg := newTestFG() r, c := newReconciler(fg) @@ -1247,7 +1557,7 @@ func TestReconcile_AppliesDeploymentUpdateWhenNoManager(t *testing.T) { t.Fatalf("update CR: %v", err) } - // No runner, no manager → reconciler must apply the update. + // No runner and no manager must still not permit a bulk existing update. if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: nn}); err != nil { t.Fatalf("second reconcile failed: %v", err) } @@ -1256,10 +1566,108 @@ func TestReconcile_AppliesDeploymentUpdateWhenNoManager(t *testing.T) { if err := c.Get(ctx, types.NamespacedName{Name: "mysql-lion-dc1", Namespace: "shared-lion"}, &d); err != nil { t.Fatalf("get deployment: %v", err) } - if d.Spec.Template.Spec.Containers[0].Image != "mysql:9.7.1" { - t.Errorf("expected image mysql:9.7.1 after reconcile (no manager), got %s", - d.Spec.Template.Spec.Containers[0].Image) + if d.Spec.Template.Spec.Containers[0].Image == "mysql:9.7.1" { + t.Errorf("existing deployment was updated outside the ordered path") + } +} + +func TestReconcile_StartupDefersExistingDeploymentsBeforeManagerRegistration(t *testing.T) { + ctx := context.Background() + fg := newTestFG() + r, c := newReconciler(fg) + nn := types.NamespacedName{Name: fg.Name, Namespace: fg.Namespace} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: nn}); err != nil { + t.Fatal(err) + } + + // The runner is wired as it is in production, but startup synchronization + // has not registered this group's manager yet. + r.Runner = &TopologyManagerRunner{managers: make(map[types.NamespacedName]*managedTopology)} + var fresh v1alpha1.MysqlFailoverGroup + if err := c.Get(ctx, nn, &fresh); err != nil { + t.Fatal(err) + } + fresh.Spec.Image = "mysql:startup-drift" + fresh.Spec.Sites = append(fresh.Spec.Sites, v1alpha1.SiteSpec{ + Name: "reader", Role: v1alpha1.SiteRoleReadOnly, + Storage: v1alpha1.StorageSpec{StorageClassName: "local-reader", Size: resource.MustParse("10Gi")}, + }) + if err := c.Update(ctx, &fresh); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: nn}); err != nil { + t.Fatal(err) + } + for _, site := range []string{"dc1", "dc2"} { + var deployment appsv1.Deployment + if err := c.Get(ctx, types.NamespacedName{Name: resourceName(fg.Name, site), Namespace: fg.Namespace}, &deployment); err != nil { + t.Fatal(err) + } + if got := deployment.Spec.Template.Spec.Containers[0].Image; got == "mysql:startup-drift" { + t.Fatalf("existing deployment %s was patched before ordered updater registration", site) + } + } + var reader appsv1.Deployment + if err := c.Get(ctx, types.NamespacedName{Name: resourceName(fg.Name, "reader"), Namespace: fg.Namespace}, &reader); err != nil { + t.Fatalf("genuinely missing reader deployment was not created: %v", err) + } +} + +func TestReconcile_HashlessLegacyDeploymentsQueueOrderedDrift(t *testing.T) { + ctx := context.Background() + fg := newTestFG() + objects := []client.Object{fg} + for _, site := range fg.Spec.Sites { + objects = append(objects, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace, + Labels: commonLabels(fg.Name, site.Name), + }, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "mysql", Image: "mysql:legacy"}}, + }}}, + }) } + r, c := newReconciler(objects...) + nn := types.NamespacedName{Name: fg.Name, Namespace: fg.Namespace} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: nn}); err != nil { + t.Fatal(err) + } + + // Bulk reconciliation must leave both hashless legacy Deployments untouched. + for _, site := range fg.Spec.Sites { + var deployment appsv1.Deployment + if err := c.Get(ctx, types.NamespacedName{Name: resourceName(fg.Name, site.Name), Namespace: fg.Namespace}, &deployment); err != nil { + t.Fatal(err) + } + if got := deployment.Annotations[specHashAnnotation]; got != "" { + t.Fatalf("hashless deployment %s was bulk-patched with hash %q", site.Name, got) + } + if got := deployment.Spec.Template.Spec.Containers[0].Image; got != "mysql:legacy" { + t.Fatalf("legacy deployment %s was bulk-patched to %q", site.Name, got) + } + } + + logger := testLogger() + runner := &TopologyManagerRunner{client: c, logger: logger, managers: make(map[types.NamespacedName]*managedTopology)} + tm := NewTopologyManager(testTopologyConfig(), []internalmysql.Checker{ + &mockMySQL{readOnly: false}, &mockMySQL{readOnly: true}, + }, NewFailoverController(logger), nil, nil, BootstrapConfig{}, newMockTainter(), platform.NewHub(logger), &mockDNS{}, logger) + runner.checkSpecDrift(ctx, fg, tm) + tm.mu.RLock() + drift := append([]string(nil), tm.specDriftSites...) + tm.mu.RUnlock() + if strings.Join(drift, ",") != "dc1,dc2" { + t.Fatalf("hashless existing deployments queued drift %v, want [dc1 dc2]", drift) + } +} + +type failingListReader struct { + client.Reader +} + +func (r failingListReader) List(context.Context, client.ObjectList, ...client.ListOption) error { + return errors.New("list failed") } // TestWaitForDeploymentRollout_ReadyReturnsImmediately verifies the wait diff --git a/internal/controller/runner.go b/internal/controller/runner.go index 1383cb3..d2bcdb8 100644 --- a/internal/controller/runner.go +++ b/internal/controller/runner.go @@ -447,10 +447,14 @@ func (r *TopologyManagerRunner) checkSpecDrift(ctx context.Context, fg *v1alpha1 Name: resourceName(fg.Name, site.Name), } if err := r.client.Get(ctx, deployNN, &deploy); err != nil { - continue // Deployment doesn't exist yet — reconciler will create it + if apierrors.IsNotFound(err) { + continue // Deployment doesn't exist yet — reconciler will create it + } + r.logger.Warn("failed to inspect deployment spec drift; preserving current drift set", "site", site.Name, "error", err) + return } liveHash := deploy.Annotations[specHashAnnotation] - if liveHash != "" && liveHash != desiredHash { + if liveHash != desiredHash { driftSites = append(driftSites, site.Name) } } @@ -477,6 +481,8 @@ func (r *TopologyManagerRunner) startManager(ctx context.Context, fg *v1alpha1.M if fg.Spec.UsesCredentials() { tlsConfigName := "" if fg.Spec.TLS != nil { + // Dial the always-published internal Service while retaining the + // client-facing hostname as TLS ServerName for existing certificates. tlsConfigName, err = mysqlTLSConfig(ctx, r.client, fg, siteServiceHost(fg.Name, site.Name, fg.Namespace)) if err != nil { for j := 0; j < i; j++ { @@ -581,6 +587,9 @@ func (r *TopologyManagerRunner) startManager(ctx context.Context, fg *v1alpha1.M r.logger.Info("restored lastFailover from CR status", "fg", nn, "lastFailover", fg.Status.LastFailover.Time) } for _, site := range fg.Status.Sites { + if site.SourceConvergenceState != "" || site.SourceHost != "" { + tm.SetSourceConvergence(site.Name, site.SourceHost, SourceConvergenceState(site.SourceConvergenceState), site.SourceConvergenceReason) + } if site.RecoveryState != recoveryStateBlocked && site.RecoveryState != recoveryStateInProgress && site.DivergentGtid == "" { continue } @@ -782,7 +791,7 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam freshFG.Status.Sites = make([]v1alpha1.SiteStatus, len(snap.Sites)) } for i, s := range snap.Sites { - freshFG.Status.Sites[i] = siteStatusFromSnapshot(s.Name, s.State, s.LastSeen, s.Replication) + freshFG.Status.Sites[i] = siteStatusFromSnapshot(s) } if !snap.LastFailover.IsZero() { @@ -809,15 +818,14 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam } // Evaluate replication health. - hasWritable := false + hasWritable := validSnapshotActiveSite(snap) != "" replicationHealthy := true for _, s := range snap.Sites { - if s.State == state.StateWritable { - hasWritable = true - } - if s.Replication != nil { - replicationHealthy = replicationHealthy && s.Replication.IORunning && s.Replication.SQLRunning + if s.Role == state.SiteRoleReadOnly || s.Name == snap.ActiveSite { + continue } + replicationHealthy = replicationHealthy && s.ReplicationHealthy && + s.SourceConvergenceState == sourceConvergenceConverged } // Update conditions using freshFG.Generation so ObservedGeneration reflects @@ -861,12 +869,12 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam } // Add replication-specific degraded conditions. - maxLagSeconds := int64(300) - if freshFG.Spec.Replication != nil && freshFG.Spec.Replication.MaxLagSeconds > 0 { - maxLagSeconds = freshFG.Spec.Replication.MaxLagSeconds - } + maxLagSeconds := freshFG.Spec.EffectiveMaxLagSeconds() for _, s := range snap.Sites { + if s.Role == state.SiteRoleReadOnly { + continue + } repl := s.Replication if repl == nil { continue @@ -882,6 +890,14 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam Message: fmt.Sprintf("Replication IO/SQL thread not running on %s", siteName), }) } + if s.SourceConvergenceState != sourceConvergenceConverged { + setCondition(&freshFG.Status.Conditions, metav1.Condition{ + Type: "Degraded", Status: metav1.ConditionTrue, + ObservedGeneration: freshFG.Generation, LastTransitionTime: now, + Reason: "ReplicationSourceMismatch", + Message: fmt.Sprintf("Replication source on %s is not the active primary", siteName), + }) + } if repl.SecondsBehindSource != nil && *repl.SecondsBehindSource > maxLagSeconds { setCondition(&freshFG.Status.Conditions, metav1.Condition{ Type: "Degraded", @@ -996,6 +1012,28 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam return nil } +func validSnapshotActiveSite(snap TopologySnapshot) string { + if snap.ActiveSite != "" { + for _, site := range snap.Sites { + if site.Name == snap.ActiveSite && site.State == state.StateWritable && site.Role != state.SiteRoleDROnly && site.Role != state.SiteRoleReadOnly { + return snap.ActiveSite + } + } + return "" + } + active := "" + for _, site := range snap.Sites { + if site.State != state.StateWritable { + continue + } + if active != "" || site.Role == state.SiteRoleDROnly || site.Role == state.SiteRoleReadOnly { + return "" + } + active = site.Name + } + return active +} + // populatePITRStatus fills freshFG.Status.PITR from the cached // per-site archiver snapshots. Aggregates across sites by taking the // oldest first-event time, newest last-event time, maximum file @@ -1185,8 +1223,12 @@ func degradedReason(snap TopologySnapshot) string { if snap.Alert == "" { return "Healthy" } - var writable, readOnly, unreachable int + var writable, readOnly, unreachable, coreSites int for _, s := range snap.Sites { + if s.Role == state.SiteRoleReadOnly { + continue + } + coreSites++ switch s.State { case state.StateWritable: writable++ @@ -1196,13 +1238,13 @@ func degradedReason(snap TopologySnapshot) string { unreachable++ } } - if len(snap.Sites) == 0 { + if coreSites == 0 { return "Alert" } switch { case writable > 1: return "SplitBrain" - case unreachable == len(snap.Sites): + case unreachable == coreSites: return "TotalLoss" case writable == 0 && readOnly > 0: return "NoPrimary" @@ -1293,19 +1335,22 @@ func (r *TopologyManagerRunner) updateBootstrappingCondition(ctx context.Context } } -func siteStatusFromSnapshot(name string, s state.SiteState, lastSeen time.Time, repl *internalmysql.ReplicaStatus) v1alpha1.SiteStatus { +func siteStatusFromSnapshot(s SiteSnapshot) v1alpha1.SiteStatus { status := v1alpha1.SiteStatus{ - Name: name, - State: s.String(), - } - if !lastSeen.IsZero() { - t := metav1.NewTime(lastSeen) + Name: s.Name, + State: s.State.String(), + SourceHost: s.SourceHost, + SourceConvergenceState: v1alpha1.SourceConvergenceState(s.SourceConvergenceState), + SourceConvergenceReason: s.SourceConvergenceReason, + } + if !s.LastSeen.IsZero() { + t := metav1.NewTime(s.LastSeen) status.LastSeen = &t } - if repl != nil { - status.Replicating = repl.IORunning && repl.SQLRunning - status.SecondsBehindSource = repl.SecondsBehindSource - status.GtidExecuted = repl.ExecutedGtidSet + if s.Replication != nil { + status.Replicating = s.ReplicationHealthy + status.SecondsBehindSource = s.Replication.SecondsBehindSource + status.GtidExecuted = s.Replication.ExecutedGtidSet } return status } @@ -1338,7 +1383,7 @@ func buildSiteDSN(baseDSN string, fg *v1alpha1.MysqlFailoverGroup, site v1alpha1 if err != nil { return "", fmt.Errorf("parse DSN: %w", err) } - parsed.Addr = fmt.Sprintf("mysql-%s-%s.%s.svc.cluster.local:%d", fg.Name, site.Name, fg.Namespace, mysqlPort) + parsed.Addr = fmt.Sprintf("%s:%d", internalSiteServiceHost(fg.Name, site.Name, fg.Namespace), mysqlPort) return parsed.FormatDSN(), nil } @@ -1350,7 +1395,7 @@ func buildSiteDSNFromCreds(username, password string, fg *v1alpha1.MysqlFailover cfg.User = username cfg.Passwd = password cfg.Net = "tcp" - cfg.Addr = fmt.Sprintf("%s:%d", siteServiceHost(fg.Name, site.Name, fg.Namespace), mysqlPort) + cfg.Addr = fmt.Sprintf("%s:%d", internalSiteServiceHost(fg.Name, site.Name, fg.Namespace), mysqlPort) cfg.Timeout = 5 * time.Second if tlsConfigName != "" { cfg.TLSConfig = tlsConfigName diff --git a/internal/controller/runner_test.go b/internal/controller/runner_test.go index 80c4f51..63685d9 100644 --- a/internal/controller/runner_test.go +++ b/internal/controller/runner_test.go @@ -210,7 +210,6 @@ func TestUpdateCRStatus_IsNotFound_NoPanic(t *testing.T) { {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, } @@ -239,7 +238,6 @@ func TestUpdateCRStatus_ExistingCR(t *testing.T) { {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, ActiveSite: "dc1", } @@ -290,7 +288,6 @@ func TestUpdateCRStatus_DeletedMidUpdate(t *testing.T) { {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, } @@ -389,8 +386,7 @@ func TestUpdateCRStatus_SetsConditions(t *testing.T) { {Name: "dc1", State: state.StateWritable}, - {Name: "dc2", State: state.StateReadOnly}, - + {Name: "dc2", State: state.StateReadOnly, ReplicationHealthy: true, SourceConvergenceState: sourceConvergenceConverged}, }, ActiveSite: "dc1", } @@ -416,6 +412,80 @@ func TestUpdateCRStatus_SetsConditions(t *testing.T) { } } +func TestUpdateCRStatus_WritableNonPromotableIsDegraded(t *testing.T) { + for _, role := range []state.SiteRole{state.SiteRoleReadOnly, state.SiteRoleDROnly} { + t.Run(string(role), func(t *testing.T) { + fg := newTestFG() + fg.Spec.Sites = append(fg.Spec.Sites, v1alpha1.SiteSpec{Name: "anomaly", Role: v1alpha1.SiteRole(role)}) + scheme := testScheme() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&v1alpha1.MysqlFailoverGroup{}). + WithObjects(fg).Build() + runner := &TopologyManagerRunner{client: c, logger: testLogger(), managers: make(map[types.NamespacedName]*managedTopology)} + nn := types.NamespacedName{Name: fg.Name, Namespace: fg.Namespace} + runner.updateCRStatus(context.Background(), nn, TopologySnapshot{ + Sites: []SiteSnapshot{ + {Name: "dc1", Role: state.SiteRolePrimaryCandidate, State: state.StateWritable}, + {Name: "dc2", Role: state.SiteRolePrimaryCandidate, State: state.StateReadOnly, ReplicationHealthy: true, SourceConvergenceState: sourceConvergenceConverged}, + {Name: "anomaly", Role: role, State: state.StateWritable, ReplicationHealthy: true, SourceConvergenceState: sourceConvergenceConverged}, + }, + DegradedReason: "Degraded", + Alert: "writable non-promotable site requires fencing (anomaly)", + }) + var updated v1alpha1.MysqlFailoverGroup + if err := c.Get(context.Background(), nn, &updated); err != nil { + t.Fatal(err) + } + var ready, degraded *metav1.Condition + for i := range updated.Status.Conditions { + condition := &updated.Status.Conditions[i] + switch condition.Type { + case "Ready": + ready = condition + case "Degraded": + degraded = condition + } + } + if ready == nil || ready.Status != metav1.ConditionFalse { + t.Fatalf("Ready condition = %+v, want False", ready) + } + if degraded == nil || degraded.Status != metav1.ConditionTrue || degraded.Reason != "Degraded" { + t.Fatalf("Degraded condition = %+v, want True/Degraded", degraded) + } + }) + } +} + +func TestUpdateOnlyStatusCallbackPreservesNoPrimaryCondition(t *testing.T) { + fg := newTestFG() + scheme := testScheme() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&v1alpha1.MysqlFailoverGroup{}). + WithObjects(fg).Build() + runner := &TopologyManagerRunner{client: c, logger: testLogger(), managers: make(map[types.NamespacedName]*managedTopology)} + tm, _, _ := newTestTopologyManager(&mockMySQL{readOnly: true}, &mockMySQL{readOnly: true}) + tm.sites[0].state = state.StateReadOnly + tm.sites[1].state = state.StateReadOnly + nn := types.NamespacedName{Name: fg.Name, Namespace: fg.Namespace} + tm.StatusCallback = func(snapshot TopologySnapshot) { runner.updateCRStatus(context.Background(), nn, snapshot) } + + tm.emitStatusSnapshot() + tm.emitStatusSnapshot() // update-completion-style callback without a transition + var updated v1alpha1.MysqlFailoverGroup + if err := c.Get(context.Background(), nn, &updated); err != nil { + t.Fatal(err) + } + for _, condition := range updated.Status.Conditions { + if condition.Type == "Degraded" { + if condition.Status != metav1.ConditionTrue || condition.Reason != "NoPrimary" { + t.Fatalf("persistent topology degradation was cleared: %+v", condition) + } + return + } + } + t.Fatal("Degraded condition not found") +} + func drainRunnerEvents(rec *record.FakeRecorder) []string { var out []string for { @@ -576,13 +646,12 @@ func TestEmitDegradedTransitionEvents_SplitBrain(t *testing.T) { runner, nn := newDegradedTestRunner(rec, fg, "Healthy") snap := TopologySnapshot{ - Alert: "SPLIT BRAIN: both sites are writable", + Alert: "SPLIT BRAIN: both sites are writable", Sites: []SiteSnapshot{ {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateWritable}, - }, } @@ -603,13 +672,12 @@ func TestEmitDegradedTransitionEvents_NoPrimary(t *testing.T) { runner, nn := newDegradedTestRunner(rec, fg, "Healthy") snap := TopologySnapshot{ - Alert: "NO PRIMARY: both sites are read-only", + Alert: "NO PRIMARY: both sites are read-only", Sites: []SiteSnapshot{ {Name: "dc1", State: state.StateReadOnly}, {Name: "dc2", State: state.StateReadOnly}, - }, } @@ -630,13 +698,12 @@ func TestEmitDegradedTransitionEvents_TotalLoss(t *testing.T) { runner, nn := newDegradedTestRunner(rec, fg, "Healthy") snap := TopologySnapshot{ - Alert: "TOTAL LOSS: both sites are unreachable", + Alert: "TOTAL LOSS: both sites are unreachable", Sites: []SiteSnapshot{ {Name: "dc1", State: state.StateUnreachable}, {Name: "dc2", State: state.StateUnreachable}, - }, } @@ -662,7 +729,6 @@ func TestEmitDegradedTransitionEvents_SiteRecovered(t *testing.T) { {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, } @@ -683,13 +749,12 @@ func TestEmitDegradedTransitionEvents_NoEventOnSameReason(t *testing.T) { runner, nn := newDegradedTestRunner(rec, fg, "SplitBrain") snap := TopologySnapshot{ - Alert: "SPLIT BRAIN: both sites are writable", + Alert: "SPLIT BRAIN: both sites are writable", Sites: []SiteSnapshot{ {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateWritable}, - }, } @@ -707,13 +772,12 @@ func TestEmitDegradedTransitionEvents_TransitionBetweenAlerts(t *testing.T) { runner, nn := newDegradedTestRunner(rec, fg, "SplitBrain") snap := TopologySnapshot{ - Alert: "TOTAL LOSS: both sites are unreachable", + Alert: "TOTAL LOSS: both sites are unreachable", Sites: []SiteSnapshot{ {Name: "dc1", State: state.StateUnreachable}, {Name: "dc2", State: state.StateUnreachable}, - }, } @@ -739,7 +803,6 @@ func TestEmitDegradedTransitionEvents_NoRecoveryEventOnFreshManager(t *testing.T {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, } @@ -763,7 +826,6 @@ func TestEmitDegradedTransitionEvents_ReplicationDoesNotCauseFalseRecovery(t *te {Name: "dc1", State: state.StateWritable}, {Name: "dc2", State: state.StateReadOnly}, - }, } runner.emitDegradedTransitionEvents(fg, nn, snap) @@ -817,7 +879,6 @@ func TestUpdateCRStatus_EmitsFailoverEvent(t *testing.T) { {Name: "dc1", State: state.StateUnreachable}, {Name: "dc2", State: state.StateWritable}, - }, ActiveSite: "dc2", LastFailover: failoverTime, diff --git a/internal/controller/source_convergence.go b/internal/controller/source_convergence.go new file mode 100644 index 0000000..396311e --- /dev/null +++ b/internal/controller/source_convergence.go @@ -0,0 +1,272 @@ +package controller + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/shipstream/bloodraven/internal/metrics" + "github.com/shipstream/bloodraven/internal/mysql" + "github.com/shipstream/bloodraven/internal/state" +) + +const sourceConvergenceOperationTimeout = 20 * time.Second + +func canonicalSourceHost(host string) string { + host = strings.ToLower(strings.TrimSpace(host)) + host = strings.TrimSuffix(host, ":3306") + return strings.TrimSuffix(host, ".") +} + +// confirmedActivePrimary returns the unique observed writable promotable site +// after directly confirming it still accepts writes. +func (tm *TopologyManager) confirmedActivePrimary(ctx context.Context) (*siteTracker, error) { + tm.mu.RLock() + name := tm.activeSiteLocked() + tm.mu.RUnlock() + if name == "" { + return nil, fmt.Errorf("no unique writable primary-candidate") + } + site := tm.getSite(name) + if site == nil || !site.isPromotable() || site.host == "" { + return nil, fmt.Errorf("active site %q is not a valid donor", name) + } + if err := tm.confirmWritable(ctx, site); err != nil { + return nil, err + } + return site, nil +} + +func (tm *TopologyManager) sourceConvergenceSuppressed() bool { + tm.mu.RLock() + defer tm.mu.RUnlock() + return tm.promotedSite != "" || tm.autoBootstrapSuppressed || tm.topologyFrozen || + tm.plannedFailoverActive || !bootstrapIdlePhase(tm.bootstrapPhase) || + (tm.updater != nil && tm.updater.IsUpdating()) +} + +// checkSourceConvergence converges followers with existing replication +// metadata directly onto the confirmed active primary. Source-less followers +// remain the responsibility of bootstrap/old-primary recovery. +func (tm *TopologyManager) checkSourceConvergence(ctx context.Context, siteRepl []*mysql.ReplicaStatus) (map[string]struct{}, bool) { + handled := make(map[string]struct{}) + active, err := tm.confirmedActivePrimary(ctx) + if err != nil { + return handled, false + } + suppressed := tm.sourceConvergenceSuppressed() + changed := false + + for i := range tm.sites { + site := &tm.sites[i] + if site.name == active.name || site.state != state.StateReadOnly { + continue + } + var repl *mysql.ReplicaStatus + if i < len(siteRepl) { + repl = siteRepl[i] + } + if repl == nil || strings.TrimSpace(repl.SourceHost) == "" { + continue + } + handled[site.name] = struct{}{} + current := canonicalSourceHost(repl.SourceHost) + expected := canonicalSourceHost(active.host) + if current == expected && repl.IORunning && repl.SQLRunning { + changed = tm.setSourceConvergence(site, repl, sourceConvergenceConverged, sourceReasonDirectSource, active.host) || changed + continue + } + + if suppressed { + changed = tm.setSourceConvergence(site, repl, sourceConvergencePending, sourceReasonSourceMismatch, active.host) || changed + continue + } + + opCtx, cancel := context.WithTimeout(ctx, sourceConvergenceOperationTimeout) + if current == expected { + err = tm.restartDirectReplica(opCtx, active, site) + } else { + err = tm.repointReplica(opCtx, active, site, repl.SourceHost) + } + if err != nil { + cancel() + stateValue, reason := sourceConvergencePending, sourceReasonMutationFailed + if strings.Contains(err.Error(), sourceReasonGTIDDiverged) { + stateValue, reason = sourceConvergenceBlocked, sourceReasonGTIDDiverged + } else if strings.Contains(err.Error(), sourceReasonProbeFailed) { + reason = sourceReasonProbeFailed + } + changed = tm.setSourceConvergence(site, repl, stateValue, reason, active.host) || changed + continue + } + + verified, verifyErr := tm.verifyDirectReplica(opCtx, site, active.host) + cancel() + if verifyErr != nil { + tm.logSourceFailure(site, active, "verify", verifyErr) + changed = tm.setSourceConvergence(site, repl, sourceConvergencePending, sourceReasonMutationFailed, active.host) || changed + continue + } + if i < len(siteRepl) { + siteRepl[i] = verified + } + tm.logger.Info("replication source convergence complete", "site", site.name, "source", active.host, "fg", tm.cfg.Name) + changed = tm.setSourceConvergence(site, verified, sourceConvergenceConverged, sourceReasonDirectSource, active.host) || changed + } + return handled, changed +} + +func (tm *TopologyManager) restartDirectReplica(ctx context.Context, active, follower *siteTracker) error { + if err := tm.ensureGTIDContained(ctx, active, follower, "pre-start-gtid"); err != nil { + return err + } + tm.logger.Info("replication source convergence started", + "site", follower.name, "activeSite", active.name, "currentSource", active.host, + "expectedSource", active.host, "fg", tm.cfg.Name) + if err := follower.mysql.StartReplica(ctx); err != nil { + tm.logSourceFailure(follower, active, "start", err) + return err + } + return nil +} + +func (tm *TopologyManager) repointReplica(ctx context.Context, active, follower *siteTracker, currentSource string) error { + if err := tm.ensureGTIDContained(ctx, active, follower, "pre-stop-gtid"); err != nil { + return err + } + tm.logger.Info("replication source convergence started", + "site", follower.name, "activeSite", active.name, "currentSource", currentSource, + "expectedSource", active.host, "fg", tm.cfg.Name) + if err := follower.mysql.StopReplica(ctx); err != nil { + tm.logSourceFailure(follower, active, "stop", err) + return err + } + if err := tm.ensureGTIDContained(ctx, active, follower, "post-stop-gtid"); err != nil { + // The unchanged channel is safe to resume when the failure was a probe; + // divergence deliberately remains stopped for operator review. + if !strings.Contains(err.Error(), sourceReasonGTIDDiverged) { + _ = follower.mysql.StartReplica(ctx) + } + return err + } + if err := follower.mysql.ChangeReplicationSource(ctx, mysql.ReplicationSourceOpts{ + Host: active.host, User: tm.bootstrapCfg.ReplUser, Password: tm.bootstrapCfg.ReplPassword, UseSSL: tm.bootstrapCfg.UseSSL, + }); err != nil { + tm.logSourceFailure(follower, active, "change-source", err) + _ = follower.mysql.StartReplica(ctx) + return err + } + if err := follower.mysql.StartReplica(ctx); err != nil { + tm.logSourceFailure(follower, active, "start", err) + return err + } + return nil +} + +func (tm *TopologyManager) ensureGTIDContained(ctx context.Context, active, follower *siteTracker, stage string) error { + activeRaw, err := active.mysql.GetGtidExecuted(ctx) + if err != nil { + tm.logSourceFailure(follower, active, stage, err) + return fmt.Errorf("%s: %w", sourceReasonProbeFailed, err) + } + followerRaw, err := follower.mysql.GetGtidExecuted(ctx) + if err != nil { + tm.logSourceFailure(follower, active, stage, err) + return fmt.Errorf("%s: %w", sourceReasonProbeFailed, err) + } + activeGTID, activeErr := mysql.ParseGTIDSet(activeRaw) + followerGTID, followerErr := mysql.ParseGTIDSet(followerRaw) + if activeErr != nil || followerErr != nil { + parseErr := activeErr + if parseErr == nil { + parseErr = followerErr + } + tm.logSourceFailure(follower, active, stage, parseErr) + return fmt.Errorf("%s: %w", sourceReasonProbeFailed, parseErr) + } + if !activeGTID.Contains(followerGTID) { + tm.logger.Warn("replication source convergence blocked", + "site", follower.name, "activeSite", active.name, "stage", stage, + "followerGtid", followerRaw, "activeGtid", activeRaw, "fg", tm.cfg.Name) + return fmt.Errorf("%s: active does not contain follower GTID", sourceReasonGTIDDiverged) + } + return nil +} + +func (tm *TopologyManager) verifyDirectReplica(ctx context.Context, follower *siteTracker, expectedHost string) (*mysql.ReplicaStatus, error) { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + rs, err := follower.mysql.ShowReplicaStatus(ctx) + if err == nil && rs != nil && rs.IORunning && rs.SQLRunning && canonicalSourceHost(rs.SourceHost) == canonicalSourceHost(expectedHost) { + return rs, nil + } + if err != nil { + lastErr = err + } else { + lastErr = fmt.Errorf("replica source or threads not converged") + } + if attempt < 2 { + time.Sleep(100 * time.Millisecond) + } + } + return nil, lastErr +} + +func (tm *TopologyManager) logSourceFailure(follower, active *siteTracker, stage string, err error) { + tm.logger.Error("replication source convergence failed", + "site", follower.name, "activeSite", active.name, "stage", stage, "error", err, "fg", tm.cfg.Name) +} + +func (tm *TopologyManager) setSourceConvergence(site *siteTracker, repl *mysql.ReplicaStatus, convergence SourceConvergenceState, reason, expectedHost string) bool { + host := "" + if repl != nil { + host = repl.SourceHost + } + serving := site.role == state.SiteRoleReadOnly && site.state == state.StateReadOnly && site.replicating && + convergence == sourceConvergenceConverged && canonicalSourceHost(host) == canonicalSourceHost(expectedHost) && + repl != nil && repl.SecondsBehindSource != nil && *repl.SecondsBehindSource <= tm.cfg.ReadOnlyMaxLagSeconds + tm.mu.Lock() + defer tm.mu.Unlock() + changed := site.sourceHost != host || site.sourceConvergenceState != convergence || + site.sourceConvergenceReason != reason || site.servingHealthy != serving + site.sourceHost = host + site.sourceConvergenceState = convergence + site.sourceConvergenceReason = reason + site.servingHealthy = serving + return changed +} + +func (tm *TopologyManager) markSourceProbeFailed(site *siteTracker) bool { + tm.mu.Lock() + defer tm.mu.Unlock() + changed := site.sourceConvergenceState != sourceConvergencePending || + site.sourceConvergenceReason != sourceReasonProbeFailed || site.servingHealthy + site.sourceConvergenceState = sourceConvergencePending + site.sourceConvergenceReason = sourceReasonProbeFailed + site.servingHealthy = false + return changed +} + +func (tm *TopologyManager) emitSourceConvergenceMetrics() { + tm.mu.RLock() + defer tm.mu.RUnlock() + active := tm.activeSiteLocked() + for i := range tm.sites { + site := &tm.sites[i] + if site.name == active { + for _, value := range metrics.AllSourceStates { + metrics.ReplicationSourceState.DeleteLabelValues(site.name, value) + } + continue + } + current := strings.ToLower(string(site.sourceConvergenceState)) + for _, value := range metrics.AllSourceStates { + v := 0.0 + if current == value { + v = 1 + } + metrics.ReplicationSourceState.WithLabelValues(site.name, value).Set(v) + } + } +} diff --git a/internal/controller/source_convergence_test.go b/internal/controller/source_convergence_test.go new file mode 100644 index 0000000..08ab792 --- /dev/null +++ b/internal/controller/source_convergence_test.go @@ -0,0 +1,136 @@ +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/shipstream/bloodraven/internal/mysql" + "github.com/shipstream/bloodraven/internal/platform" + "github.com/shipstream/bloodraven/internal/state" +) + +const convergenceTestGTID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-10" + +func newConvergenceManager(t *testing.T, roles []state.SiteRole, checkers ...*mockMySQL) *TopologyManager { + t.Helper() + sites := make([]SiteTopologyConfig, len(checkers)) + interfaces := make([]mysql.Checker, len(checkers)) + for i := range checkers { + name := []string{"primary", "follower-a", "follower-b"}[i] + sites[i] = SiteTopologyConfig{Name: name, Role: roles[i], Host: "mysql-" + name + "-internal.ns.svc.cluster.local"} + interfaces[i] = checkers[i] + } + tm := NewTopologyManager(TopologyConfig{Name: "fg", Sites: sites, ReadOnlyMaxLagSeconds: 30}, interfaces, + NewFailoverController(testLogger()), nil, nil, + BootstrapConfig{ReplUser: "repl", ReplPassword: "secret"}, newMockTainter(), platform.NewHub(testLogger()), nil, testLogger()) + tm.sites[0].state = state.StateWritable + for i := 1; i < len(tm.sites); i++ { + tm.sites[i].state = state.StateReadOnly + } + return tm +} + +func TestCanonicalSourceHost(t *testing.T) { + for _, input := range []string{ + "mysql-primary-internal.ns.svc.cluster.local", + " MYSQL-PRIMARY-INTERNAL.NS.SVC.CLUSTER.LOCAL. ", + "mysql-primary-internal.ns.svc.cluster.local:3306", + "MYSQL-PRIMARY-INTERNAL.NS.SVC.CLUSTER.LOCAL.:3306", + } { + if got := canonicalSourceHost(input); got != "mysql-primary-internal.ns.svc.cluster.local" { + t.Fatalf("canonicalSourceHost(%q) = %q", input, got) + } + } +} + +func TestSourceConvergence_MultipleFollowers(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID} + lag := int64(0) + followerA := &mockMySQL{readOnly: true, gtidExecuted: convergenceTestGTID, replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "intermediate", SecondsBehindSource: &lag}} + followerB := &mockMySQL{readOnly: true, gtidExecuted: convergenceTestGTID, replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "follower-a", SecondsBehindSource: &lag}} + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleDROnly, state.SiteRoleReadOnly}, primary, followerA, followerB) + repl := []*mysql.ReplicaStatus{nil, followerA.replicaStatusVal, followerB.replicaStatusVal} + handled, changed := tm.checkSourceConvergence(context.Background(), repl) + if !changed || len(handled) != 2 { + t.Fatalf("changed=%v handled=%v", changed, handled) + } + for i, follower := range []*mockMySQL{followerA, followerB} { + if follower.stopReplicaCalls != 1 || follower.changeSourceCalls != 1 || follower.startReplicaCalls != 1 || follower.resetReplicaCalls != 0 { + t.Fatalf("follower %d calls stop=%d change=%d start=%d reset=%d", i, follower.stopReplicaCalls, follower.changeSourceCalls, follower.startReplicaCalls, follower.resetReplicaCalls) + } + } + if tm.sites[2].sourceConvergenceState != sourceConvergenceConverged { + t.Fatalf("reader source state = %s", tm.sites[2].sourceConvergenceState) + } +} + +func TestSourceConvergence_DivergenceBeforeAndAfterStop(t *testing.T) { + t.Run("pre stop", func(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID} + follower := &mockMySQL{readOnly: true, gtidExecuted: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb:1", replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "wrong"}} + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, primary, follower) + tm.checkSourceConvergence(context.Background(), []*mysql.ReplicaStatus{nil, follower.replicaStatusVal}) + if follower.stopReplicaCalls != 0 || tm.sites[1].sourceConvergenceReason != sourceReasonGTIDDiverged { + t.Fatalf("stop=%d reason=%s", follower.stopReplicaCalls, tm.sites[1].sourceConvergenceReason) + } + }) + t.Run("post stop race", func(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID} + follower := &mockMySQL{readOnly: true, gtidSequence: []string{"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-5", "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb:1"}, replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "wrong"}} + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, primary, follower) + tm.checkSourceConvergence(context.Background(), []*mysql.ReplicaStatus{nil, follower.replicaStatusVal}) + if follower.stopReplicaCalls != 1 || follower.changeSourceCalls != 0 || follower.startReplicaCalls != 0 { + t.Fatalf("calls stop=%d change=%d start=%d", follower.stopReplicaCalls, follower.changeSourceCalls, follower.startReplicaCalls) + } + }) +} + +func TestSourceConvergence_MutationFailuresAndSuppression(t *testing.T) { + for _, tc := range []struct { + name string + configure func(*mockMySQL) + }{ + {name: "stop", configure: func(f *mockMySQL) { f.stopReplicaErr = errors.New("stop") }}, + {name: "change", configure: func(f *mockMySQL) { f.changeSourceErr = errors.New("change") }}, + {name: "start", configure: func(f *mockMySQL) { f.startReplicaErr = errors.New("start") }}, + {name: "verify", configure: func(f *mockMySQL) { f.changeDoesNotUpdate = true }}, + } { + t.Run(tc.name, func(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID} + follower := &mockMySQL{readOnly: true, gtidExecuted: convergenceTestGTID, replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "wrong"}} + tc.configure(follower) + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, primary, follower) + tm.checkSourceConvergence(context.Background(), []*mysql.ReplicaStatus{nil, follower.replicaStatusVal}) + if tm.sites[1].sourceConvergenceState != sourceConvergencePending || follower.resetReplicaCalls != 0 { + t.Fatalf("state=%s reset=%d", tm.sites[1].sourceConvergenceState, follower.resetReplicaCalls) + } + }) + } + + for _, tc := range []struct { + name string + suppress func(*TopologyManager) + }{ + {name: "pending promotion", suppress: func(tm *TopologyManager) { tm.promotedSite = "primary" }}, + {name: "bootstrap", suppress: func(tm *TopologyManager) { tm.bootstrapPhase = BootstrapPhaseCloning }}, + {name: "restore", suppress: func(tm *TopologyManager) { tm.autoBootstrapSuppressed = true }}, + {name: "topology freeze", suppress: func(tm *TopologyManager) { tm.topologyFrozen = true }}, + {name: "planned failover", suppress: func(tm *TopologyManager) { tm.plannedFailoverActive = true }}, + {name: "ordered update", suppress: func(tm *TopologyManager) { + tm.updater = NewUpdateController(NewFailoverController(testLogger()), testLogger()) + tm.updater.updating = true + }}, + } { + t.Run(tc.name, func(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID} + follower := &mockMySQL{readOnly: true, gtidExecuted: convergenceTestGTID, replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "wrong"}} + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, primary, follower) + tc.suppress(tm) + tm.checkSourceConvergence(context.Background(), []*mysql.ReplicaStatus{nil, follower.replicaStatusVal}) + if follower.stopReplicaCalls != 0 || tm.sites[1].sourceConvergenceState != sourceConvergencePending { + t.Fatalf("suppressed convergence mutated follower") + } + }) + } +} diff --git a/internal/controller/topology.go b/internal/controller/topology.go index 92d2819..b514bc6 100644 --- a/internal/controller/topology.go +++ b/internal/controller/topology.go @@ -43,10 +43,12 @@ type TopologyConfig struct { // spec.sites. Sites []SiteTopologyConfig - PollInterval int64 // nanoseconds - FailureThreshold int - RecoveryThreshold int - FailoverCooldown int64 // nanoseconds, default 5m + PollInterval int64 // nanoseconds + FailureThreshold int + RecoveryThreshold int + FailoverCooldown int64 // nanoseconds, default 5m + MaxLagSeconds int64 + ReadOnlyMaxLagSeconds int64 // SitePriorities is the ordered list of primary-candidate site // names that win split-brain ties. An empty slice means manual @@ -105,12 +107,33 @@ func (c TopologyConfig) PollIntervalDuration() time.Duration { // SiteSnapshot captures one site's observation at a point in time. type SiteSnapshot struct { - Name string - Role state.SiteRole - State state.SiteState - LastSeen time.Time - Replication *mysql.ReplicaStatus // nil if site is primary or unreachable -} + Name string + Role state.SiteRole + State state.SiteState + LastSeen time.Time + Replication *mysql.ReplicaStatus // nil if site is primary or unreachable + SourceHost string + SourceConvergenceState SourceConvergenceState + SourceConvergenceReason string + ServingHealthy bool + ReplicationHealthy bool +} + +// SourceConvergenceState is the topology manager's internal representation of +// the persisted per-follower source state. +type SourceConvergenceState string + +const ( + sourceConvergenceConverged SourceConvergenceState = "Converged" + sourceConvergencePending SourceConvergenceState = "Pending" + sourceConvergenceBlocked SourceConvergenceState = "Blocked" + + sourceReasonDirectSource = "DirectSource" + sourceReasonSourceMismatch = "SourceMismatch" + sourceReasonProbeFailed = "ProbeFailed" + sourceReasonMutationFailed = "MutationFailed" + sourceReasonGTIDDiverged = "GTIDDiverged" +) // TopologySnapshot captures the topology state at a point in time. // It is passed to the StatusCallback after each poll cycle that @@ -154,8 +177,12 @@ type siteTracker struct { // threads running and a configured source host, observed for replicatingStreak // consecutive poll ticks. Used by checkUpdate to refuse ordered updates against // a stale standby whose super_read_only=ON but replication is not actually running. - replicating bool - replicatingStreak int + replicating bool + replicatingStreak int + sourceHost string + sourceConvergenceState SourceConvergenceState + sourceConvergenceReason string + servingHealthy bool } // isHealthyReplica reports whether the site is a read-only replica with debounced @@ -324,6 +351,10 @@ type SiteStatusEntry struct { Name string `json:"name"` Role string `json:"role,omitempty"` State string `json:"state"` + SourceHost string `json:"sourceHost,omitempty"` + SourceConvergenceState string `json:"sourceConvergenceState,omitempty"` + SourceConvergenceReason string `json:"sourceConvergenceReason,omitempty"` + ServingHealthy bool `json:"servingHealthy,omitempty"` RecoveryState string `json:"recoveryState,omitempty"` DivergentGtid string `json:"divergentGtid,omitempty"` DivergentTransactionCount *int64 `json:"divergentTransactionCount,omitempty"` @@ -432,20 +463,33 @@ func (tm *TopologyManager) SetRecoveryInProgress(site string) { tm.recoveryDivergentCount = 0 } -// activeSiteLocked returns the name of the single writable site, or "" -// if zero or more than one site is writable. Must be called with tm.mu -// held. +// SetSourceConvergence restores persisted source state across an operator +// restart. Live polling remains authoritative and will clear or retry it. +func (tm *TopologyManager) SetSourceConvergence(site, host string, convergence SourceConvergenceState, reason string) { + tm.mu.Lock() + defer tm.mu.Unlock() + if s := tm.getSite(site); s != nil { + s.sourceHost = host + s.sourceConvergenceState = convergence + s.sourceConvergenceReason = reason + } +} + +// activeSiteLocked returns the unique writable primary-candidate. Any second +// writable site, including a reader or dr-only anomaly, makes authority +// ambiguous and returns empty. Must be called with tm.mu held. func (tm *TopologyManager) activeSiteLocked() string { - var name string - var count int + var active *siteTracker for i := range tm.sites { if tm.sites[i].state == state.StateWritable { - name = tm.sites[i].name - count++ + if active != nil { + return "" + } + active = &tm.sites[i] } } - if count == 1 { - return name + if active != nil && active.isPromotable() { + return active.name } return "" } @@ -456,9 +500,13 @@ func (tm *TopologyManager) Status() StatusResponse { sites := make([]SiteStatusEntry, len(tm.sites)) for i := range tm.sites { sites[i] = SiteStatusEntry{ - Name: tm.sites[i].name, - Role: string(tm.sites[i].role), - State: tm.sites[i].state.String(), + Name: tm.sites[i].name, + Role: string(tm.sites[i].role), + State: tm.sites[i].state.String(), + SourceHost: tm.sites[i].sourceHost, + SourceConvergenceState: string(tm.sites[i].sourceConvergenceState), + SourceConvergenceReason: tm.sites[i].sourceConvergenceReason, + ServingHealthy: tm.sites[i].servingHealthy, } if tm.recoveryPendingSite == tm.sites[i].name { sites[i].RecoveryState = tm.recoveryStateLocked() @@ -483,12 +531,25 @@ func (tm *TopologyManager) effectiveActiveSiteLocked() string { if active := tm.activeSiteLocked(); active != "" { return active } - if tm.pendingPromotionFreshLocked() { + if tm.pendingPromotionFreshLocked() && tm.pendingPromotionUnambiguousLocked() { return tm.promotedSite } return "" } +func (tm *TopologyManager) pendingPromotionUnambiguousLocked() bool { + target := tm.getSite(tm.promotedSite) + if target == nil || !target.isPromotable() { + return false + } + for i := range tm.sites { + if tm.sites[i].state == state.StateWritable && tm.sites[i].name != tm.promotedSite { + return false + } + } + return true +} + func (tm *TopologyManager) pendingPromotionFreshLocked() bool { if tm.promotedSite == "" { return false @@ -650,6 +711,12 @@ func (tm *TopologyManager) Poll(ctx context.Context) { newStates := make([]state.SiteState, len(tm.sites)) for i := range tm.sites { newStates[i] = tm.computeState(&tm.sites[i], results[i].readOnly, results[i].err) + // A successful writable observation on a non-promotable site is an + // immediate safety fact. Do not debounce authority invalidation or + // fencing behind the normal recovery threshold. + if results[i].err == nil && !results[i].readOnly && !tm.sites[i].isPromotable() { + newStates[i] = state.StateWritable + } } // Update lastSeen and state under the lock. @@ -682,6 +749,10 @@ func (tm *TopologyManager) Poll(ctx context.Context) { } } + // A writable non-promotable site is never authoritative. Enforce this on + // every poll so a failed fence is retried without waiting for a transition. + fencedNonPromotable := tm.fenceWritableNonPromotableSites(ctx) + // Check for pending promotion confirmation: DNS was flipped after successful // promotion and writable confirmation; this block verifies the promoted site // is still writable and clears the guard flag to allow future failovers. @@ -691,13 +762,11 @@ func (tm *TopologyManager) Poll(ctx context.Context) { } tm.mu.Unlock() - // Cross-site evaluation (only on state transitions to avoid repeated actions). - var alertMsg, degradedReason string + // Evaluate every poll so all status snapshots carry the current topology + // condition. Mutating cross-site actions remain transition-driven. + action := state.EvalCrossSite(tm.observations(), tm.cfg.SitePriorities) + alertMsg := action.Alert if anyTransition { - observations := tm.observations() - action := state.EvalCrossSite(observations, tm.cfg.SitePriorities) - alertMsg = action.Alert - degradedReason = action.Reason tm.applyCrossSiteAction(ctx, action) } @@ -723,7 +792,9 @@ func (tm *TopologyManager) Poll(ctx context.Context) { // Check replication status on every read-only site. siteRepl := make([]*mysql.ReplicaStatus, len(tm.sites)) + replicaProbeFailed := make(map[string]struct{}) replicationChanged := false + sourceProbeChanged := false const replicatingStreakThreshold = 2 for i := range tm.sites { if tm.sites[i].state != state.StateReadOnly { @@ -742,6 +813,8 @@ func (tm *TopologyManager) Poll(ctx context.Context) { replicationChanged = true } tm.mu.Unlock() + sourceProbeChanged = tm.markSourceProbeFailed(&tm.sites[i]) || sourceProbeChanged + replicaProbeFailed[tm.sites[i].name] = struct{}{} continue } siteRepl[i] = rs @@ -763,8 +836,16 @@ func (tm *TopologyManager) Poll(ctx context.Context) { tm.mu.Unlock() } + // Converge every configured follower directly onto the unique confirmed + // active primary before old-primary recovery considers source-less sites. + handledConvergence, convergenceChanged := tm.checkSourceConvergence(ctx, siteRepl) + for site := range replicaProbeFailed { + handledConvergence[site] = struct{}{} + } + convergenceChanged = convergenceChanged || sourceProbeChanged + // Check if old primary recovery is needed. - recoveryChanged := tm.checkRecovery(ctx, siteRepl, alertMsg, degradedReason) + recoveryChanged := tm.checkRecoveryWithConvergence(ctx, siteRepl, handledConvergence) // Restore writability on the promoted primary if its sidecar fenced // it back to read-only and no writable site remains (poll-driven so @@ -826,6 +907,7 @@ func (tm *TopologyManager) Poll(ctx context.Context) { metrics.ReplicationRunning.DeleteLabelValues(name, "sql") } } + tm.emitSourceConvergenceMetrics() // Notify the status callback on any state change, recovery event, or // update event — and additionally whenever a prior /status write was @@ -835,7 +917,7 @@ func (tm *TopologyManager) Poll(ctx context.Context) { tm.mu.RLock() statusRetry := tm.statusWriteFailed tm.mu.RUnlock() - statusChanged := anyTransition || replicationChanged || recoveryChanged || recloneStarted || autoCloneStarted || updateStarted || reasserted + statusChanged := anyTransition || fencedNonPromotable || replicationChanged || convergenceChanged || recoveryChanged || recloneStarted || autoCloneStarted || updateStarted || reasserted if (statusChanged || statusRetry) && tm.StatusCallback != nil { var snapshot TopologySnapshot if statusRetry && !statusChanged { @@ -845,7 +927,7 @@ func (tm *TopologyManager) Poll(ctx context.Context) { } tm.mu.RUnlock() } else { - snapshot = tm.buildSnapshot(siteRepl, alertMsg, degradedReason) + snapshot = tm.buildSnapshot(siteRepl) } tm.mu.Lock() tm.statusRetrySnapshot = &snapshot @@ -1037,9 +1119,13 @@ func (tm *TopologyManager) observations() []state.SiteObservation { } // buildSnapshot produces a TopologySnapshot from current tracker state. -// Callers must provide the most recent siteRepl values and any alert -// computed this cycle. -func (tm *TopologyManager) buildSnapshot(siteRepl []*mysql.ReplicaStatus, alert, degradedReason string) TopologySnapshot { +// Callers must provide the most recent siteRepl values. +func (tm *TopologyManager) buildSnapshot(siteRepl []*mysql.ReplicaStatus) TopologySnapshot { + // Snapshot callbacks also originate outside Poll (for example update + // completion and status retries). Derive topology health from current + // trackers every time so those callbacks cannot clear persistent alerts. + action := state.EvalCrossSite(tm.observations(), tm.cfg.SitePriorities) + alert, degradedReason := action.Alert, action.Reason tm.mu.RLock() defer tm.mu.RUnlock() @@ -1050,11 +1136,16 @@ func (tm *TopologyManager) buildSnapshot(siteRepl []*mysql.ReplicaStatus, alert, repl = siteRepl[i] } sites[i] = SiteSnapshot{ - Name: tm.sites[i].name, - Role: tm.sites[i].role, - State: tm.sites[i].state, - LastSeen: tm.sites[i].lastSeen, - Replication: repl, + Name: tm.sites[i].name, + Role: tm.sites[i].role, + State: tm.sites[i].state, + LastSeen: tm.sites[i].lastSeen, + Replication: repl, + SourceHost: tm.sites[i].sourceHost, + SourceConvergenceState: tm.sites[i].sourceConvergenceState, + SourceConvergenceReason: tm.sites[i].sourceConvergenceReason, + ServingHealthy: tm.sites[i].servingHealthy, + ReplicationHealthy: tm.sites[i].replicating, } } @@ -1177,6 +1268,9 @@ func (tm *TopologyManager) taintSelector(site *siteTracker) string { } func (tm *TopologyManager) applyPerSiteAction(ctx context.Context, site *siteTracker, action state.Action) { + if site.role == state.SiteRoleReadOnly || site.taintSelector == "" { + return + } if action.Taint != nil { selector := tm.taintSelector(site) if err := tm.tainter.SetTaint(ctx, selector, tm.cfg.Name, *action.Taint); err != nil { @@ -1191,6 +1285,23 @@ func (tm *TopologyManager) applyPerSiteAction(ctx context.Context, site *siteTra } } +func (tm *TopologyManager) fenceWritableNonPromotableSites(ctx context.Context) bool { + attempted := false + for i := range tm.sites { + site := &tm.sites[i] + if site.state != state.StateWritable || site.isPromotable() { + continue + } + attempted = true + if err := site.mysql.SetSuperReadOnly(ctx, true); err != nil { + tm.logger.Error("failed to fence writable non-promotable site", "site", site.name, "role", site.role, "error", err) + } else { + tm.logger.Warn("fenced writable non-promotable site", "site", site.name, "role", site.role) + } + } + return attempted +} + func (tm *TopologyManager) applyCrossSiteAction(ctx context.Context, action state.CrossSiteAction) { if action.Alert != "" { tm.logger.Warn("ALERT", "message", action.Alert) @@ -1769,52 +1880,89 @@ func (tm *TopologyManager) checkUpdate(ctx context.Context) bool { return false } - // Identify the active site and each healthy standby. - var activeName string - var activeChecker mysql.Checker - var standbyName string - var standbyChecker mysql.Checker tm.mu.RLock() - for i := range tm.sites { - switch tm.sites[i].state { - case state.StateWritable: - if activeName != "" { - // Split-brain — let that path recover before touching updates. - tm.mu.RUnlock() - return false - } - activeName = tm.sites[i].name - activeChecker = tm.sites[i].mysql - case state.StateReadOnly: - if tm.sites[i].isHealthyReplica() && standbyName == "" { - standbyName = tm.sites[i].name - standbyChecker = tm.sites[i].mysql - } - } - } + activeName := tm.activeSiteLocked() + driftSites := append([]string(nil), tm.specDriftSites...) tm.mu.RUnlock() - if activeName == "" || standbyName == "" { + if activeName == "" { return false } - - tm.mu.RLock() - driftSites := tm.specDriftSites - tm.mu.RUnlock() if len(driftSites) == 0 { return false } + drifted := make(map[string]bool, len(driftSites)) + for _, name := range driftSites { + drifted[name] = true + } + activeSite := tm.getSite(activeName) + active := UpdateTarget{ + Name: activeName, Host: activeSite.host, Checker: activeSite.mysql, Promotable: true, Drifted: drifted[activeName], + ReplUser: tm.bootstrapCfg.ReplUser, ReplPassword: tm.bootstrapCfg.ReplPassword, UseSSL: tm.bootstrapCfg.UseSSL, + } + followers := make([]UpdateTarget, 0, len(tm.sites)-1) + for i := range tm.sites { + site := &tm.sites[i] + if site.name == activeName { + continue + } + followers = append(followers, UpdateTarget{ + Name: site.name, Host: site.host, Checker: site.mysql, Promotable: site.isPromotable(), + Drifted: drifted[site.name], ExpectedSource: activeSite.host, + }) + } + if drifted[activeName] { + if tm.bootstrapCfg.ReplUser == "" { + return false + } + haveStandby := false + for i := range tm.sites { + if tm.sites[i].name != activeName && tm.sites[i].isPromotable() && tm.sites[i].isHealthyReplica() { + haveStandby = true + break + } + } + if !haveStandby { + return false + } + } tm.logger.Info("ordered update: spec drift detected, starting ordered update", - "driftSites", driftSites, "active", activeName, "standby", standbyName) + "driftSites", driftSites, "active", activeName) applyUpdate := tm.ApplyUpdate go func() { - if err := tm.updater.Execute(ctx, activeName, standbyName, standbyChecker, activeChecker, applyUpdate); err != nil { + processed, err := tm.updater.ExecuteTargets(ctx, active, followers, applyUpdate, func(target, promotionGTID string) { + targetSite := tm.getSite(target) + now := tm.clock.Now() + tm.mu.Lock() + tm.promotionGtidExecuted = promotionGTID + tm.promotedSite = target + tm.promotedAt = now + tm.lastFailover = now + tm.lastFailoverTarget = target + tm.mu.Unlock() + metrics.FailoversTotal.WithLabelValues(target).Inc() + if targetSite != nil { + if dnsErr := tm.applyDNS(ctx, target, targetSite.lbIP); dnsErr != nil { + tm.logger.Error("ordered update: DNS flip failed after handoff", "site", target, "error", dnsErr) + } + } + }) + if err != nil { tm.logger.Error("ordered update failed", "error", err) } - // Clear drift sites after update completes (success or failure). tm.mu.Lock() - tm.specDriftSites = nil + completed := make(map[string]bool, len(processed)) + for _, name := range processed { + completed[name] = true + } + remaining := tm.specDriftSites[:0] + for _, name := range tm.specDriftSites { + if !completed[name] { + remaining = append(remaining, name) + } + } + tm.specDriftSites = remaining tm.mu.Unlock() // Trigger a status callback to clear the Updating condition. if tm.StatusCallback != nil { @@ -1842,7 +1990,7 @@ func (tm *TopologyManager) emitStatusSnapshot() { } } } - tm.StatusCallback(tm.buildSnapshot(siteRepl, "", "")) + tm.StatusCallback(tm.buildSnapshot(siteRepl)) } // SetSpecDriftSites records which sites have spec drift (Deployment hash != desired hash). @@ -1890,84 +2038,119 @@ type userSchemaChecker interface { // multiple sites are empty the operator clones one per poll cycle. // Returns ("", "") when no eligible pair exists. func (tm *TopologyManager) detectEmptySite(ctx context.Context) (donor, empty string) { - // Every site must be reachable. + active, err := tm.confirmedActivePrimary(ctx) + if err != nil { + // A freshly recreated empty candidate starts writable. Identify and + // fence that empty recipient before establishing unique donor authority. + return tm.detectAndFenceEmptyWritableSite(ctx) + } + + // Preserve the core-site reachability guard, but a missing reader must not + // block recovery of a promotable or dr-only site. for i := range tm.sites { + if tm.sites[i].role == state.SiteRoleReadOnly { + continue + } switch tm.sites[i].state { case state.StateUnreachable, state.StateUnknown: return "", "" } } - replStatus := make([]*mysql.ReplicaStatus, len(tm.sites)) - gtidSets := make([]mysql.GTIDSet, len(tm.sites)) - hasUserSchemas := make([]bool, len(tm.sites)) + // Candidates are populated before dr-only sites, and readers last. + for _, role := range []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleDROnly, state.SiteRoleReadOnly} { + for i := range tm.sites { + site := &tm.sites[i] + if site.name == active.name || site.role != role || + (site.state != state.StateWritable && site.state != state.StateReadOnly) { + continue + } + rs, probeErr := site.mysql.ShowReplicaStatus(ctx) + if probeErr != nil { + if site.role == state.SiteRoleReadOnly { + continue + } + return "", "" + } + raw, probeErr := site.mysql.GetGtidExecuted(ctx) + if probeErr != nil { + if site.role == state.SiteRoleReadOnly { + continue + } + return "", "" + } + gtid, probeErr := mysql.ParseGTIDSet(raw) + if probeErr != nil { + return "", "" + } + hasSchemas := !gtid.IsEmpty() + if checker, ok := site.mysql.(userSchemaChecker); ok { + hasSchemas, probeErr = checker.HasUserSchemas(ctx) + if probeErr != nil { + if site.role == state.SiteRoleReadOnly { + continue + } + return "", "" + } + } + freshInitialized := site.state == state.StateReadOnly && !hasSchemas + if rs == nil && (gtid.IsEmpty() || freshInitialized) { + return active.name, site.name + } + } + } + return "", "" +} + +func (tm *TopologyManager) detectAndFenceEmptyWritableSite(ctx context.Context) (string, string) { + var donor, recipient *siteTracker for i := range tm.sites { - rs, err := tm.sites[i].mysql.ShowReplicaStatus(ctx) - if err != nil { + site := &tm.sites[i] + if site.state != state.StateWritable || site.role == state.SiteRoleReadOnly { + continue + } + rs, err := site.mysql.ShowReplicaStatus(ctx) + if err != nil || rs != nil { return "", "" } - replStatus[i] = rs - - raw, err := tm.sites[i].mysql.GetGtidExecuted(ctx) + raw, err := site.mysql.GetGtidExecuted(ctx) if err != nil { return "", "" } - parsed, err := mysql.ParseGTIDSet(raw) + gtid, err := mysql.ParseGTIDSet(raw) if err != nil { return "", "" } - gtidSets[i] = parsed - - if checker, ok := tm.sites[i].mysql.(userSchemaChecker); ok { - hasUserSchema, err := checker.HasUserSchemas(ctx) + hasSchemas := !gtid.IsEmpty() + if checker, ok := site.mysql.(userSchemaChecker); ok { + hasSchemas, err = checker.HasUserSchemas(ctx) if err != nil { return "", "" } - hasUserSchemas[i] = hasUserSchema - } else { - hasUserSchemas[i] = !parsed.IsEmpty() - } - } - - // Locate the writable site that should donate data. Prefer a donor - // with GTIDs, but allow an empty writable donor so a freshly reset - // cluster can still bootstrap replication after a replica PVC wipe. - // We only clone from a writable site to avoid copying from a stale - // replica. - donorIdx := -1 - for i := range tm.sites { - if tm.sites[i].state == state.StateWritable && !gtidSets[i].IsEmpty() { - donorIdx = i - break } - } - if donorIdx < 0 { - for i := range tm.sites { - if tm.sites[i].state == state.StateWritable { - donorIdx = i - break + if hasSchemas { + if donor != nil || !site.isPromotable() { + return "", "" } + donor = site + } else { + if recipient != nil { + return "", "" + } + recipient = site } } - if donorIdx < 0 { + if donor == nil || recipient == nil || donor.host == "" { return "", "" } - - // Locate any empty, reachable non-donor site. - for i := range tm.sites { - if i == donorIdx { - continue - } - reachable := tm.sites[i].state == state.StateWritable || tm.sites[i].state == state.StateReadOnly - if !reachable { - continue - } - freshInitialized := tm.sites[i].state == state.StateReadOnly && !hasUserSchemas[i] - if replStatus[i] == nil && (gtidSets[i].IsEmpty() || freshInitialized) { - return tm.sites[donorIdx].name, tm.sites[i].name - } + if err := recipient.mysql.SetSuperReadOnly(ctx, true); err != nil { + return "", "" } - return "", "" + readOnly, err := recipient.mysql.CheckReadOnly(ctx) + if err != nil || !readOnly || tm.confirmWritable(ctx, donor) != nil { + return "", "" + } + return donor.name, recipient.name } // selectSeedSite determines which site should be seeded as the initial @@ -2005,6 +2188,25 @@ func (tm *TopologyManager) startBootstrap(ctx context.Context) { tm.emitBootstrapStatus() return } + seedSite := tm.getSite(seed) + for i := range tm.sites { + if tm.sites[i].name == seed { + continue + } + if err := tm.sites[i].mysql.SetSuperReadOnly(ctx, true); err != nil { + tm.logger.Error("fresh-deploy bootstrap: failed to fence non-seed site", "site", tm.sites[i].name, "error", err) + return + } + readOnly, err := tm.sites[i].mysql.CheckReadOnly(ctx) + if err != nil || !readOnly { + tm.logger.Error("fresh-deploy bootstrap: non-seed fence not confirmed", "site", tm.sites[i].name, "error", err) + return + } + } + if seedSite == nil || !seedSite.isPromotable() || seedSite.host == "" || tm.confirmWritable(ctx, seedSite) != nil { + tm.logger.Error("fresh-deploy bootstrap: seed authority could not be confirmed", "site", seed) + return + } // Pick any non-seed site as the first clone recipient. We prefer // primary-candidate sites first so that post-fresh-deploy there is @@ -2059,6 +2261,13 @@ func (tm *TopologyManager) startBootstrapByName(ctx context.Context, donor, reci tm.emitBootstrapStatus() return } + if source != "fresh-deploy" { + confirmed, err := tm.confirmedActivePrimary(ctx) + if err != nil || confirmed.name != donorSite.name { + tm.logger.Error("bootstrap aborted: donor is not the confirmed active primary", "donor", donor, "error", err) + return + } + } tm.mu.Lock() tm.bootstrapPhase = BootstrapPhaseCloning @@ -2076,7 +2285,18 @@ func (tm *TopologyManager) startBootstrapByName(ctx context.Context, donor, reci go func() { allowSkipClone := source != "reclone" - err := tm.runBootstrap(ctx, donorSite.mysql, recipientSite.mysql, donorSite.host, recipient, allowSkipClone) + var err error + if source != "fresh-deploy" { + confirmed, confirmErr := tm.confirmedActivePrimary(ctx) + if confirmErr != nil { + err = fmt.Errorf("bootstrap donor authority lost before clone: %w", confirmErr) + } else if confirmed.name != donorSite.name { + err = fmt.Errorf("bootstrap donor authority moved from %s to %s", donorSite.name, confirmed.name) + } + } + if err == nil { + err = tm.runBootstrap(ctx, donorSite.mysql, recipientSite.mysql, donorSite.host, recipient, allowSkipClone) + } tm.mu.Lock() if err != nil { tm.bootstrapPhase = BootstrapPhaseFailed @@ -2084,6 +2304,10 @@ func (tm *TopologyManager) startBootstrapByName(ctx context.Context, donor, reci tm.logger.Error("bootstrap failed", "source", source, "error", err) } else { tm.bootstrapPhase = BootstrapPhaseDone + recipientSite.sourceHost = "" + recipientSite.sourceConvergenceState = "" + recipientSite.sourceConvergenceReason = "" + recipientSite.servingHealthy = false tm.logger.Info("bootstrap completed successfully", "source", source) } tm.mu.Unlock() @@ -2237,7 +2461,11 @@ func (tm *TopologyManager) recoveryStateLocked() string { // non-active site and pick the first one that needs recovery; multiple // divergent sites are reported sequentially across poll cycles. // Returns true if recovery state changed this cycle. -func (tm *TopologyManager) checkRecovery(ctx context.Context, siteRepl []*mysql.ReplicaStatus, alert, degradedReason string) bool { +func (tm *TopologyManager) checkRecovery(ctx context.Context, siteRepl []*mysql.ReplicaStatus) bool { + return tm.checkRecoveryWithConvergence(ctx, siteRepl, nil) +} + +func (tm *TopologyManager) checkRecoveryWithConvergence(ctx context.Context, siteRepl []*mysql.ReplicaStatus, convergenceHandled map[string]struct{}) bool { if tm.isBootstrapping() { return false } @@ -2254,17 +2482,21 @@ func (tm *TopologyManager) checkRecovery(ctx context.Context, siteRepl []*mysql. return false } - // Find the active (writable) site; refuse to proceed on split-brain. + // Recovery is destructive to replication metadata, so it uses the same + // unique, directly confirmed, promotable authority gate as convergence and + // clone/reclone paths. + active, err := tm.confirmedActivePrimary(ctx) + if err != nil { + return false + } activeIdx := -1 for i := range tm.sites { - if tm.sites[i].state == state.StateWritable { - if activeIdx != -1 { - return false - } + if &tm.sites[i] == active { activeIdx = i + break } } - if activeIdx == -1 { + if activeIdx < 0 { return false } @@ -2274,6 +2506,9 @@ func (tm *TopologyManager) checkRecovery(ctx context.Context, siteRepl []*mysql. continue } other := &tm.sites[i] + if _, handled := convergenceHandled[other.name]; handled { + continue + } if other.state != state.StateReadOnly { continue } @@ -2299,7 +2534,7 @@ func (tm *TopologyManager) checkRecovery(ctx context.Context, siteRepl []*mysql. } } // Read-only with no active replication after a prior failover: start recovery. - return tm.initiateRecovery(ctx, i, activeIdx, siteRepl, alert, degradedReason) + return tm.initiateRecovery(ctx, i, activeIdx, siteRepl) } return false } @@ -2479,6 +2714,9 @@ func (tm *TopologyManager) clearHealthyRecoverySite(siteRepl []*mysql.ReplicaSta if i >= len(siteRepl) || !replicaStatusHealthy(siteRepl[i]) { return false } + if tm.sites[i].sourceConvergenceState != sourceConvergenceConverged { + return false + } tm.mu.Lock() tm.recoveryPendingSite = "" tm.recoveryState = "" @@ -2500,7 +2738,7 @@ func replicaStatusHealthy(repl *mysql.ReplicaStatus) bool { // initiateRecovery fences the old primary, compares GTID sets, and either // auto-recovers (no divergence) or blocks with metadata (divergence). // Returns true if recovery state changed. -func (tm *TopologyManager) initiateRecovery(ctx context.Context, oldPrimaryIdx, newPrimaryIdx int, siteRepl []*mysql.ReplicaStatus, alert, degradedReason string) bool { +func (tm *TopologyManager) initiateRecovery(ctx context.Context, oldPrimaryIdx, newPrimaryIdx int, siteRepl []*mysql.ReplicaStatus) bool { oldPrimary := &tm.sites[oldPrimaryIdx] newPrimary := &tm.sites[newPrimaryIdx] @@ -2549,7 +2787,7 @@ func (tm *TopologyManager) initiateRecovery(ctx context.Context, oldPrimaryIdx, // early write is the durable handoff for operator restarts that happen // inside the STOP/RESET/CHANGE/START sequence. if tm.StatusCallback != nil { - tm.StatusCallback(tm.buildSnapshot(siteRepl, alert, degradedReason)) + tm.StatusCallback(tm.buildSnapshot(siteRepl)) } tm.executeRecovery(ctx, oldPrimaryIdx, newPrimaryIdx) return true @@ -2631,14 +2869,8 @@ func (tm *TopologyManager) checkReclone(ctx context.Context) bool { return false } - var donor *siteTracker - for i := range tm.sites { - if tm.sites[i].state == state.StateWritable { - donor = &tm.sites[i] - break - } - } - if donor == nil { + donor, donorErr := tm.confirmedActivePrimary(ctx) + if donorErr != nil { tm.logger.Error("reclone requested but no writable primary found") return false } diff --git a/internal/controller/topology_test.go b/internal/controller/topology_test.go index 82739e5..eecb54d 100644 --- a/internal/controller/topology_test.go +++ b/internal/controller/topology_test.go @@ -3,8 +3,10 @@ package controller import ( "context" "errors" + "fmt" "log/slog" "os" + "strings" "sync" "testing" "time" @@ -18,20 +20,29 @@ import ( // --- Mock MySQL --- type mockMySQL struct { - mu sync.Mutex - readOnly bool - err error - promoted bool - replicaStatusVal *mysql.ReplicaStatus - replicaStatusErr error - gtidExecuted string - gtidExecutedErr error - hasUserSchemas *bool - userSchemasErr error - stopReplicaCalls int - resetReplicaCalls int - changeSourceCalls int - startReplicaCalls int + mu sync.Mutex + readOnly bool + err error + promoted bool + replicaStatusVal *mysql.ReplicaStatus + replicaStatusErr error + gtidExecuted string + gtidExecutedErr error + hasUserSchemas *bool + userSchemasErr error + stopReplicaCalls int + resetReplicaCalls int + changeSourceCalls int + startReplicaCalls int + stopReplicaErr error + changeSourceErr error + startReplicaErr error + gtidSequence []string + gtidCalls int + changeDoesNotUpdate bool + superReadOnlyCalls int + superReadOnlyErr error + clonePrimaryHost string } func testBoolPtr(v bool) *bool { @@ -54,13 +65,150 @@ func (m *mockMySQL) Promote(_ context.Context) error { func (m *mockMySQL) Close() error { return nil } -func (m *mockMySQL) SetSuperReadOnly(_ context.Context, _ bool) error { return nil } +func (m *mockMySQL) SetSuperReadOnly(_ context.Context, on bool) error { + m.mu.Lock() + defer m.mu.Unlock() + m.superReadOnlyCalls++ + if m.superReadOnlyErr != nil { + return m.superReadOnlyErr + } + if on { + m.readOnly = true + } + return nil +} + +func TestReadOnlyRoleSafety(t *testing.T) { + primaryA := &mockMySQL{readOnly: true} + primaryB := &mockMySQL{readOnly: true} + reader := &mockMySQL{readOnly: false} + tm := newConvergenceManager(t, + []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, + primaryA, primaryB, reader) + tm.sites[0].state = state.StateReadOnly + tm.sites[1].state = state.StateReadOnly + tm.sites[2].state = state.StateWritable + if got := tm.activeSiteLocked(); got != "" { + t.Fatalf("writable reader became active: %q", got) + } + if !tm.fenceWritableNonPromotableSites(context.Background()) || reader.superReadOnlyCalls != 1 { + t.Fatalf("reader fence calls = %d", reader.superReadOnlyCalls) + } + taint := true + tm.applyPerSiteAction(context.Background(), &tm.sites[2], state.Action{Taint: &taint}) + if got := len(tm.tainter.(*mockTainter).taints); got != 0 { + t.Fatalf("reader caused %d taint operations", got) + } +} + +func TestPoll_ReplicaStatusProbeFailureInterlocksRecovery(t *testing.T) { + primary := &mockMySQL{readOnly: false, gtidExecuted: convergenceTestGTID} + follower := &mockMySQL{ + readOnly: true, + gtidExecuted: convergenceTestGTID, + replicaStatusErr: errors.New("temporary replica probe failure"), + } + tm := newConvergenceManager(t, + []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRolePrimaryCandidate}, + primary, follower, + ) + tm.lastFailoverTarget = "primary" + + tm.Poll(context.Background()) + if got := tm.sites[1].sourceConvergenceState; got != sourceConvergencePending { + t.Fatalf("source state = %q, want Pending", got) + } + if got := tm.sites[1].sourceConvergenceReason; got != sourceReasonProbeFailed { + t.Fatalf("source reason = %q, want ProbeFailed", got) + } + if follower.stopReplicaCalls != 0 || follower.resetReplicaCalls != 0 || follower.changeSourceCalls != 0 || follower.startReplicaCalls != 0 { + t.Fatalf("probe failure mutated replication: stop=%d reset=%d change=%d start=%d", + follower.stopReplicaCalls, follower.resetReplicaCalls, follower.changeSourceCalls, follower.startReplicaCalls) + } + + follower.replicaStatusErr = nil + follower.replicaStatusVal = &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "old-primary"} + tm.Poll(context.Background()) + if got := tm.sites[1].sourceConvergenceState; got != sourceConvergenceConverged { + t.Fatalf("source state after successful retry = %q, want Converged", got) + } + if follower.stopReplicaCalls != 1 || follower.changeSourceCalls != 1 || follower.startReplicaCalls != 1 || follower.resetReplicaCalls != 0 { + t.Fatalf("retry calls stop=%d reset=%d change=%d start=%d", + follower.stopReplicaCalls, follower.resetReplicaCalls, follower.changeSourceCalls, follower.startReplicaCalls) + } +} + +func TestRecoveryRejectsWritableNonPromotableAuthority(t *testing.T) { + for _, role := range []state.SiteRole{state.SiteRoleReadOnly, state.SiteRoleDROnly} { + t.Run(string(role), func(t *testing.T) { + candidate := &mockMySQL{readOnly: true, gtidExecuted: convergenceTestGTID} + anomaly := &mockMySQL{readOnly: false, gtidExecuted: convergenceTestGTID} + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, role}, candidate, anomaly) + tm.sites[0].state = state.StateReadOnly + tm.sites[1].state = state.StateWritable + tm.lastFailoverTarget = "follower-a" + + if tm.checkRecoveryWithConvergence(context.Background(), []*mysql.ReplicaStatus{nil, nil}, nil) { + t.Fatal("recovery changed state using non-promotable authority") + } + if candidate.gtidCalls != 0 || anomaly.gtidCalls != 0 || candidate.stopReplicaCalls != 0 || candidate.resetReplicaCalls != 0 || candidate.changeSourceCalls != 0 || candidate.startReplicaCalls != 0 { + t.Fatalf("recovery probed or mutated against %s authority", role) + } + }) + } +} + +func TestPoll_FirstWritableNonPromotableObservationFencesImmediately(t *testing.T) { + for _, role := range []state.SiteRole{state.SiteRoleReadOnly, state.SiteRoleDROnly} { + for _, fenceFails := range []bool{false, true} { + name := fmt.Sprintf("%s/fence-fails-%t", role, fenceFails) + t.Run(name, func(t *testing.T) { + primary := &mockMySQL{readOnly: false} + anomaly := &mockMySQL{readOnly: false} + if fenceFails { + anomaly.superReadOnlyErr = errors.New("fence failed") + } + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, role}, primary, anomaly) + tm.sites[1].state = state.StateReadOnly + tm.cfg.RecoveryThreshold = 3 + var snapshot TopologySnapshot + tm.StatusCallback = func(s TopologySnapshot) { snapshot = s } + + tm.Poll(context.Background()) + if anomaly.superReadOnlyCalls != 1 { + t.Fatalf("first writable observation made %d fence calls, want 1", anomaly.superReadOnlyCalls) + } + if tm.sites[1].state != state.StateWritable || tm.activeSiteLocked() != "" { + t.Fatalf("anomaly did not immediately invalidate authority: state=%s active=%q", tm.sites[1].state, tm.activeSiteLocked()) + } + if snapshot.DegradedReason != "Degraded" { + t.Fatalf("degraded reason = %q, want Degraded", snapshot.DegradedReason) + } + }) + } + } +} + +func TestEmitStatusSnapshotPreservesPersistentTopologyDegradation(t *testing.T) { + tm, _, _ := newTestTopologyManager(&mockMySQL{readOnly: true}, &mockMySQL{readOnly: true}) + tm.sites[0].state = state.StateReadOnly + tm.sites[1].state = state.StateReadOnly + var snapshot TopologySnapshot + tm.StatusCallback = func(s TopologySnapshot) { snapshot = s } + + // This is the callback path used by update completion; no state transition + // occurs while the current no-primary topology remains unchanged. + tm.emitStatusSnapshot() + if snapshot.DegradedReason != "NoPrimary" || snapshot.Alert == "" { + t.Fatalf("update-only snapshot cleared degradation: %+v", snapshot) + } +} func (m *mockMySQL) KillAppConnections(_ context.Context) (int, error) { return 0, nil } func (m *mockMySQL) StopReplica(_ context.Context) error { m.mu.Lock() defer m.mu.Unlock() m.stopReplicaCalls++ - return nil + return m.stopReplicaErr } func (m *mockMySQL) ResetReplicaAll(_ context.Context) error { m.mu.Lock() @@ -79,16 +227,33 @@ func (m *mockMySQL) ShowReplicaStatus(_ context.Context) (*mysql.ReplicaStatus, defer m.mu.Unlock() return m.replicaStatusVal, m.replicaStatusErr } -func (m *mockMySQL) ChangeReplicationSource(_ context.Context, _ mysql.ReplicationSourceOpts) error { +func (m *mockMySQL) ChangeReplicationSource(_ context.Context, opts mysql.ReplicationSourceOpts) error { m.mu.Lock() defer m.mu.Unlock() m.changeSourceCalls++ + if m.changeSourceErr != nil { + return m.changeSourceErr + } + if m.changeDoesNotUpdate { + return nil + } + if m.replicaStatusVal == nil { + m.replicaStatusVal = &mysql.ReplicaStatus{} + } + m.replicaStatusVal.SourceHost = opts.Host return nil } func (m *mockMySQL) StartReplica(_ context.Context) error { m.mu.Lock() defer m.mu.Unlock() m.startReplicaCalls++ + if m.startReplicaErr != nil { + return m.startReplicaErr + } + if m.replicaStatusVal != nil { + m.replicaStatusVal.IORunning = true + m.replicaStatusVal.SQLRunning = true + } return nil } func (m *mockMySQL) StartReplicaSQLThread(_ context.Context) error { return nil } @@ -98,6 +263,14 @@ func (m *mockMySQL) WaitForRelayLogDrain(_ context.Context, _ time.Duration) err func (m *mockMySQL) GetGtidExecuted(_ context.Context) (string, error) { m.mu.Lock() defer m.mu.Unlock() + if len(m.gtidSequence) > 0 { + idx := m.gtidCalls + if idx >= len(m.gtidSequence) { + idx = len(m.gtidSequence) - 1 + } + m.gtidCalls++ + return m.gtidSequence[idx], m.gtidExecutedErr + } return m.gtidExecuted, m.gtidExecutedErr } func (m *mockMySQL) HasUserSchemas(_ context.Context) (bool, error) { @@ -110,7 +283,11 @@ func (m *mockMySQL) HasUserSchemas(_ context.Context) (bool, error) { } func (m *mockMySQL) EnsureClonePlugin(_ context.Context) error { return nil } func (m *mockMySQL) SetCloneDonorList(_ context.Context, _ string) error { return nil } -func (m *mockMySQL) CloneInstance(_ context.Context, _, _, _ string, _ bool, _ int) error { + +func (m *mockMySQL) CloneInstance(_ context.Context, _, primaryHost, _ string, _ bool, _ int) error { + m.mu.Lock() + defer m.mu.Unlock() + m.clonePrimaryHost = primaryHost return nil } @@ -227,8 +404,8 @@ func testTopologyConfig() TopologyConfig { return TopologyConfig{ Name: "lion", Sites: []SiteTopologyConfig{ - {Name: "dc1", Zone: "lion-dc1", LBIP: "1.1.1.1", Role: state.SiteRolePrimaryCandidate, TaintSelector: taintSelector("dc1")}, - {Name: "dc2", Zone: "lion-dc2", LBIP: "2.2.2.2", Role: state.SiteRolePrimaryCandidate, TaintSelector: taintSelector("dc2")}, + {Name: "dc1", Zone: "lion-dc1", LBIP: "1.1.1.1", Role: state.SiteRolePrimaryCandidate, TaintSelector: taintSelector("dc1"), Host: "mysql-dc1"}, + {Name: "dc2", Zone: "lion-dc2", LBIP: "2.2.2.2", Role: state.SiteRolePrimaryCandidate, TaintSelector: taintSelector("dc2"), Host: "mysql-dc2"}, }, PollInterval: int64(50 * time.Millisecond), FailureThreshold: 3, @@ -332,6 +509,25 @@ func TestNormalSite0Primary(t *testing.T) { } } +func TestStatusIncludesSourceConvergence(t *testing.T) { + tm, _, _ := newTestTopologyManager(&mockMySQL{readOnly: false}, &mockMySQL{readOnly: true}) + tm.SetSourceConvergence("dc2", "mysql-lion-dc1-internal.ns.svc.cluster.local", sourceConvergenceConverged, "") + tm.mu.Lock() + tm.sites[1].servingHealthy = true + tm.mu.Unlock() + + site := tm.Status().Sites[1] + if got, want := site.SourceHost, "mysql-lion-dc1-internal.ns.svc.cluster.local"; got != want { + t.Fatalf("sourceHost = %q, want %q", got, want) + } + if got, want := site.SourceConvergenceState, string(sourceConvergenceConverged); got != want { + t.Fatalf("sourceConvergenceState = %q, want %q", got, want) + } + if !site.ServingHealthy { + t.Fatal("servingHealthy = false, want true") + } +} + func TestFailoverSite0Down(t *testing.T) { site0 := &mockMySQL{readOnly: false} site1 := &mockMySQL{readOnly: true} @@ -856,7 +1052,7 @@ func TestSnapshotActiveSiteDoesNotUsePendingPromotion(t *testing.T) { tm.promotedAt = tm.clock.Now() tm.mu.Unlock() - snap := tm.buildSnapshot(nil, "", "") + snap := tm.buildSnapshot(nil) if snap.ActiveSite != "" { t.Errorf("expected CR snapshot activeSite to reflect observed writable site only, got %q", snap.ActiveSite) } @@ -1191,6 +1387,35 @@ func TestReclone_HappyPath(t *testing.T) { } } +func TestReclone_ReadOnlyRecipientUsesActivePrimary(t *testing.T) { + primary := &mockMySQL{readOnly: false} + reader := &mockMySQL{readOnly: true} + tm, _, _ := newTestTopologyManagerWithBootstrap(primary, reader) + tm.autoBootstrapSuppressed = true + tm.sites[1].role = state.SiteRoleReadOnly + tm.cfg.Sites[1].Role = state.SiteRoleReadOnly + pollN(tm, 2) + + tm.SetRecloneSite("dc2") + if !tm.checkReclone(context.Background()) { + t.Fatal("expected reader reclone to start") + } + tm.mu.RLock() + source := tm.bootstrapSource + tm.mu.RUnlock() + var primaryHost string + deadline := time.Now().Add(time.Second) + for primaryHost == "" && time.Now().Before(deadline) { + reader.mu.Lock() + primaryHost = reader.clonePrimaryHost + reader.mu.Unlock() + time.Sleep(time.Millisecond) + } + if source != "reclone" || primaryHost != "mysql-dc1" { + t.Fatalf("bootstrap = source %q primary host %q, want reclone/mysql-dc1", source, primaryHost) + } +} + func TestReclone_CannotReclonePrimary(t *testing.T) { site0 := &mockMySQL{readOnly: false} site1 := &mockMySQL{readOnly: true} @@ -1265,12 +1490,13 @@ func TestCheckRecovery_ClearsHealthyRecoverySiteWithoutLastFailoverTarget(t *tes tm.recoveryDivergentGtid = "aaaa:50-55" tm.recoveryDivergentCount = 6 tm.lastFailoverTarget = "" + tm.sites[1].sourceConvergenceState = sourceConvergenceConverged tm.mu.Unlock() changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{ nil, {IORunning: true, SQLRunning: true, SourceHost: "mysql-dc1"}, - }, "", "") + }) if !changed { t.Fatal("checkRecovery should report a status change after clearing healthy recovery state") } @@ -1297,12 +1523,13 @@ func TestCheckRecovery_ClearsHealthyRecoverySiteWithoutReplicationCredentials(t tm.recoveryState = recoveryStateInProgress tm.lastFailoverTarget = "dc1" tm.bootstrapCfg = BootstrapConfig{} + tm.sites[1].sourceConvergenceState = sourceConvergenceConverged tm.mu.Unlock() changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{ nil, {IORunning: true, SQLRunning: true, SourceHost: "mysql-dc1"}, - }, "", "") + }) if !changed { t.Fatal("checkRecovery should clear healthy recovery state even when retry credentials are unavailable") } @@ -1329,7 +1556,7 @@ func TestCheckRecovery_PersistsInProgressAndSuppressesImmediateRetry(t *testing. snapshots = append(snapshots, s) } - changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}, "", "") + changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}) if !changed { t.Fatal("expected recovery to start") } @@ -1340,7 +1567,7 @@ func TestCheckRecovery_PersistsInProgressAndSuppressesImmediateRetry(t *testing. t.Fatalf("expected RecoveryInProgress snapshot before recovery, got %#v", snapshots) } - changed = tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}, "", "") + changed = tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}) if changed { t.Fatal("recovery should not retry during stabilization window") } @@ -1349,7 +1576,7 @@ func TestCheckRecovery_PersistsInProgressAndSuppressesImmediateRetry(t *testing. } clk.Advance(recoveryRetryDelay + time.Second) - changed = tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}, "", "") + changed = tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}) if !changed { t.Fatal("expected recovery retry after stabilization window") } @@ -1367,7 +1594,7 @@ func TestCheckRecovery_RestoredInProgressRetriesImmediatelyWhenUnhealthy(t *test setRecoveredTopology(tm) tm.SetRecoveryInProgress("dc2") - changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}, "", "") + changed := tm.checkRecovery(context.Background(), []*mysql.ReplicaStatus{nil, nil}) if !changed { t.Fatal("expected restored in-progress recovery to retry when unhealthy") } @@ -1714,7 +1941,7 @@ func TestPoll_ClearsReplicatingWhenReplicationStopped(t *testing.T) { site1.replicaStatusVal = &mysql.ReplicaStatus{ IORunning: false, SQLRunning: false, - SourceHost: "dc1", + SourceHost: "mysql-dc1", } site1.mu.Unlock() @@ -1762,12 +1989,12 @@ func TestPoll_StatusCallbackFiresWhenReplicationBecomesHealthy(t *testing.T) { site1.mu.Unlock() pollN(tm, 1) - if callbacks != 0 { - t.Fatalf("first healthy replication tick is debounce-only, got %d callbacks", callbacks) + if callbacks != 1 { + t.Fatalf("first healthy tick should persist source convergence, got %d callbacks", callbacks) } pollN(tm, 1) - if callbacks != 1 { - t.Fatalf("replication-only health change should trigger one callback, got %d", callbacks) + if callbacks != 2 { + t.Fatalf("debounced replication health should trigger a second callback, got %d", callbacks) } if captured.Sites[1].State != state.StateReadOnly { t.Fatalf("site1 state = %s, want read-only", captured.Sites[1].State) @@ -1827,10 +2054,9 @@ func TestPoll_StatusCallbackFiresWhenReplicaStatusErrorsAfterHealthy(t *testing. } } -// TestCheckUpdate_DefersWhenStandbyNotReplicating verifies the issue #46 gate: -// even with spec drift and both sites in the right states, checkUpdate must NOT -// start an ordered update against a stale standby. -func TestCheckUpdate_DefersWhenStandbyNotReplicating(t *testing.T) { +// TestCheckUpdate_RetainsUnhealthyStandbyDrift verifies an unsafe follower is +// retained by the ordered plan without applying its Deployment update. +func TestCheckUpdate_RetainsUnhealthyStandbyDrift(t *testing.T) { site0 := &mockMySQL{readOnly: false} site1 := &mockMySQL{ readOnly: true, @@ -1840,23 +2066,79 @@ func TestCheckUpdate_DefersWhenStandbyNotReplicating(t *testing.T) { // Attach an UpdateController and ApplyUpdate callback so checkUpdate doesn't // early-return on nil dependencies. tm.updater = NewUpdateController(NewFailoverController(testLogger()), testLogger()) - tm.ApplyUpdate = func(_ context.Context, _ string) error { - t.Fatal("ApplyUpdate must not be called when standby is not replicating") + applied := make(chan string, 1) + tm.ApplyUpdate = func(_ context.Context, site string) error { + applied <- site return nil } - // Poll enough to settle states (site1 -> StateReadOnly with RecoveryThreshold=2). pollN(tm, 3) // Drift on standby — the classic trigger for an ordered update. tm.SetSpecDriftSites([]string{"dc2"}) + done := make(chan struct{}) + tm.StatusCallback = func(TopologySnapshot) { close(done) } started := tm.checkUpdate(context.Background()) - if started { - t.Fatal("checkUpdate must defer when the standby is not a healthy replica") + if !started { + t.Fatal("checkUpdate must start so independently safe targets can be processed") + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ordered plan did not complete") } - if tm.updater.IsUpdating() { - t.Error("UpdateController must not enter updating state on deferred start") + select { + case site := <-applied: + t.Fatalf("unsafe standby %s was updated", site) + default: + } + tm.mu.RLock() + drift := append([]string(nil), tm.specDriftSites...) + tm.mu.RUnlock() + if len(drift) != 1 || drift[0] != "dc2" { + t.Fatalf("unhealthy standby drift was not retained: %v", drift) + } +} + +func TestCheckUpdate_ProcessesHealthyCandidateBeforeUnhealthyReader(t *testing.T) { + primary := &mockMySQL{readOnly: false} + candidate := &mockMySQL{readOnly: true, replicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "mysql-primary-internal.ns.svc.cluster.local", + }} + reader := &mockMySQL{readOnly: true, replicaStatusVal: &mysql.ReplicaStatus{ + IORunning: false, SQLRunning: false, SourceHost: "mysql-primary-internal.ns.svc.cluster.local", + }} + tm := newConvergenceManager(t, []state.SiteRole{ + state.SiteRolePrimaryCandidate, state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly, + }, primary, candidate, reader) + tm.sites[1].replicating = true + tm.updater = NewUpdateController(NewFailoverController(testLogger()), testLogger()) + var applied []string + tm.ApplyUpdate = func(_ context.Context, site string) error { + applied = append(applied, site) + return nil + } + tm.SetSpecDriftSites([]string{"follower-a", "follower-b"}) + done := make(chan struct{}) + tm.StatusCallback = func(TopologySnapshot) { close(done) } + + if !tm.checkUpdate(context.Background()) { + t.Fatal("ordered plan did not start") + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ordered plan did not finish") + } + if got := strings.Join(applied, ","); got != "follower-a" { + t.Fatalf("applied = %q, want healthy candidate only", got) + } + tm.mu.RLock() + drift := append([]string(nil), tm.specDriftSites...) + tm.mu.RUnlock() + if len(drift) != 1 || drift[0] != "follower-b" { + t.Fatalf("remaining drift = %v, want unhealthy reader only", drift) } } diff --git a/internal/controller/updater.go b/internal/controller/updater.go index 76a332d..e298b47 100644 --- a/internal/controller/updater.go +++ b/internal/controller/updater.go @@ -44,6 +44,19 @@ type UpdateController struct { updating bool } +// UpdateTarget describes one site in an N-site ordered rollout. +type UpdateTarget struct { + Name string + Host string + Checker mysql.Checker + Promotable bool + Drifted bool + ExpectedSource string + ReplUser string + ReplPassword string + UseSSL bool +} + // NewUpdateController creates a new UpdateController. func NewUpdateController(failover *FailoverController, logger *slog.Logger) *UpdateController { return &UpdateController{ @@ -152,6 +165,136 @@ func (u *UpdateController) Execute(ctx context.Context, return nil } +// ExecuteTargets updates every drifted follower sequentially, then updates the +// active site only after handing authority to a healthy promotable standby. +// The returned names completed successfully and may be removed from drift. +func (u *UpdateController) ExecuteTargets(ctx context.Context, active UpdateTarget, followers []UpdateTarget, + applyUpdate func(context.Context, string) error, onPromoted func(string, string)) (processed []string, err error) { + u.mu.Lock() + if u.updating { + u.mu.Unlock() + return nil, fmt.Errorf("update already in progress") + } + u.updating = true + u.mu.Unlock() + defer func() { + u.mu.Lock() + u.updating = false + u.phase = UpdatePhaseNone + u.mu.Unlock() + }() + + for _, follower := range followers { + if !follower.Drifted { + continue + } + if err := u.requireDirectReplica(ctx, follower); err != nil { + u.logger.Warn("ordered update: retaining unsafe follower drift", "site", follower.Name, "error", err) + continue + } + u.setPhase(UpdatePhaseUpdateReplica) + u.logger.Info("ordered update: updating follower", "site", follower.Name) + if err := applyUpdate(ctx, follower.Name); err != nil { + return processed, fmt.Errorf("update follower %s: %w", follower.Name, err) + } + u.setPhase(UpdatePhaseWaitReplica) + if err := u.waitForReplicaReadyExpected(ctx, follower.Checker, follower.ExpectedSource, 5*time.Minute); err != nil { + return processed, fmt.Errorf("wait for follower %s: %w", follower.Name, err) + } + processed = append(processed, follower.Name) + } + + if !active.Drifted { + u.setPhase(UpdatePhaseComplete) + return processed, nil + } + + var handoff *UpdateTarget + for i := range followers { + if !followers[i].Promotable { + continue + } + if err := u.requireDirectReplica(ctx, followers[i]); err == nil { + handoff = &followers[i] + break + } + } + if handoff == nil { + return processed, fmt.Errorf("no healthy promotable standby available for active-site update") + } + if handoff.Host == "" { + return processed, fmt.Errorf("promotable standby %s has no replication host", handoff.Name) + } + + u.setPhase(UpdatePhaseFailover) + promotionGTID, err := u.failover.Execute(ctx, handoff.Checker, active.Checker, handoff.Name) + if err != nil { + return processed, fmt.Errorf("failover during update: %w", err) + } + readOnly, err := handoff.Checker.CheckReadOnly(ctx) + if err != nil || readOnly { + return processed, fmt.Errorf("updated standby %s was not confirmed writable after handoff", handoff.Name) + } + if onPromoted != nil { + onPromoted(handoff.Name, promotionGTID) + } + u.setPhase(UpdatePhaseUpdateOldPrimary) + if err := applyUpdate(ctx, active.Name); err != nil { + return processed, fmt.Errorf("update old active %s: %w", active.Name, err) + } + if err := u.failover.RecoverOldPrimary(ctx, active.Checker, handoff.Host, active.ReplUser, active.ReplPassword, active.UseSSL); err != nil { + return processed, fmt.Errorf("configure old active %s as replica of %s: %w", active.Name, handoff.Name, err) + } + u.setPhase(UpdatePhaseWaitOldPrimary) + if err := u.waitForReplicaReadyExpected(ctx, active.Checker, handoff.Host, 5*time.Minute); err != nil { + return processed, fmt.Errorf("wait for old active %s to replicate from %s: %w", active.Name, handoff.Name, err) + } + processed = append(processed, active.Name) + u.setPhase(UpdatePhaseComplete) + return processed, nil +} + +func (u *UpdateController) requireDirectReplica(ctx context.Context, target UpdateTarget) error { + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ro, err := target.Checker.CheckReadOnly(probeCtx) + if err != nil { + return fmt.Errorf("precondition: follower %s probe failed: %w", target.Name, err) + } + if !ro { + return fmt.Errorf("precondition: follower %s is writable", target.Name) + } + rs, err := target.Checker.ShowReplicaStatus(probeCtx) + if err != nil || rs == nil || !rs.IORunning || !rs.SQLRunning || canonicalSourceHost(rs.SourceHost) != canonicalSourceHost(target.ExpectedSource) { + return fmt.Errorf("precondition: follower %s is not directly replicating from the active primary", target.Name) + } + return nil +} + +func (u *UpdateController) waitForReplicaReadyExpected(ctx context.Context, checker mysql.Checker, expected string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + readOnly, roleErr := checker.CheckReadOnly(ctx) + rs, err := checker.ShowReplicaStatus(ctx) + if roleErr == nil && readOnly && err == nil && rs != nil { + if canonicalSourceHost(rs.SourceHost) == canonicalSourceHost(expected) && (!rs.IORunning || !rs.SQLRunning) { + _ = checker.StartReplica(ctx) + } + if rs.IORunning && rs.SQLRunning && canonicalSourceHost(rs.SourceHost) == canonicalSourceHost(expected) { + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for direct replication") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(u.tickInterval): + } + } +} + func (u *UpdateController) setPhase(p UpdatePhase) { u.mu.Lock() u.phase = p diff --git a/internal/controller/updater_test.go b/internal/controller/updater_test.go index 008a935..de9d84c 100644 --- a/internal/controller/updater_test.go +++ b/internal/controller/updater_test.go @@ -58,6 +58,157 @@ func TestUpdateController_ExecuteOrder(t *testing.T) { } } +func TestUpdateController_ExecuteTargetsMultipleFollowersWithoutFailover(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + active := &testutil.FakeMySQL{ReadOnlyVal: false} + healthy := func() *testutil.FakeMySQL { + return &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "primary-internal", + }} + } + var order []string + processed, err := uc.ExecuteTargets(context.Background(), UpdateTarget{Name: "primary", Checker: active}, []UpdateTarget{ + {Name: "candidate", Checker: healthy(), Promotable: true, Drifted: true, ExpectedSource: "primary-internal"}, + {Name: "reader", Checker: healthy(), Drifted: true, ExpectedSource: "primary-internal"}, + }, func(_ context.Context, name string) error { order = append(order, name); return nil }, nil) + if err != nil { + t.Fatal(err) + } + if strings.Join(order, ",") != "candidate,reader" || strings.Join(processed, ",") != "candidate,reader" { + t.Fatalf("order=%v processed=%v", order, processed) + } + if active.Promoted { + t.Fatal("follower-only drift caused failover") + } +} + +func TestUpdateController_ExecuteTargetsNeverHandsOffToReader(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + active := &testutil.FakeMySQL{ReadOnlyVal: false} + reader := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "primary-internal", + }} + var applied []string + _, err := uc.ExecuteTargets(context.Background(), UpdateTarget{Name: "primary", Checker: active, Drifted: true}, []UpdateTarget{ + {Name: "reader", Checker: reader, Drifted: true, ExpectedSource: "primary-internal"}, + }, func(_ context.Context, name string) error { applied = append(applied, name); return nil }, nil) + if err == nil || strings.Contains(strings.Join(applied, ","), "primary") { + t.Fatalf("err=%v applied=%v", err, applied) + } +} + +func TestUpdateController_ExecuteTargetsSkipsUnhealthyReader(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + active := &testutil.FakeMySQL{ReadOnlyVal: false} + candidate := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "primary-internal", + }} + reader := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: false, SQLRunning: false, SourceHost: "primary-internal", + }} + var applied []string + processed, err := uc.ExecuteTargets(context.Background(), UpdateTarget{Name: "primary", Checker: active}, []UpdateTarget{ + {Name: "candidate", Checker: candidate, Promotable: true, Drifted: true, ExpectedSource: "primary-internal"}, + {Name: "reader", Checker: reader, Drifted: true, ExpectedSource: "primary-internal"}, + }, func(_ context.Context, name string) error { + applied = append(applied, name) + return nil + }, nil) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(applied, ","); got != "candidate" { + t.Fatalf("applied = %q, want candidate", got) + } + if got := strings.Join(processed, ","); got != "candidate" { + t.Fatalf("processed = %q, want candidate; reader drift must remain", got) + } +} + +func TestUpdateController_ExecuteTargetsActiveLastRequiresDirectReplica(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + uc.tickInterval = time.Millisecond + oldPrimary := &testutil.FakeMySQL{ReadOnlyVal: false} + standby := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "old-primary-internal", + }} + reader := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: false, SQLRunning: false, SourceHost: "old-primary-internal", + }} + var order []string + processed, err := uc.ExecuteTargets(context.Background(), UpdateTarget{ + Name: "old-primary", Host: "old-primary-internal", Checker: oldPrimary, Drifted: true, + }, []UpdateTarget{ + {Name: "standby", Host: "new-primary-internal", Checker: standby, Promotable: true, Drifted: true, ExpectedSource: "old-primary-internal"}, + {Name: "reader", Host: "reader-internal", Checker: reader, Drifted: true, ExpectedSource: "old-primary-internal"}, + }, func(_ context.Context, name string) error { + order = append(order, name) + if name == "old-primary" { + oldPrimary.ReplicaStatusVal = &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "new-primary-internal"} + } + return nil + }, nil) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(order, ","); got != "standby,old-primary" { + t.Fatalf("update order = %q", got) + } + if got := strings.Join(processed, ","); got != "standby,old-primary" { + t.Fatalf("processed = %q", got) + } + if !oldPrimary.ResetReplicaAllCalled || oldPrimary.ChangeReplicationOpts.Host != "new-primary-internal" || !oldPrimary.ReplicaStarted { + t.Fatalf("old primary was not configured from new authority: reset=%t opts=%+v started=%t", + oldPrimary.ResetReplicaAllCalled, oldPrimary.ChangeReplicationOpts, oldPrimary.ReplicaStarted) + } +} + +func TestWaitForReplicaReadyExpectedRequiresReadOnlyDirectThreads(t *testing.T) { + for _, tc := range []struct { + name string + checker *testutil.FakeMySQL + }{ + {name: "writable", checker: &testutil.FakeMySQL{ReadOnlyVal: false, ReplicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "new-primary"}}}, + {name: "stopped threads", checker: &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{SourceHost: "new-primary"}}}, + {name: "wrong source", checker: &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "old-primary"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + uc := NewUpdateController(NewFailoverController(testutil.TestLogger()), testutil.TestLogger()) + uc.tickInterval = time.Millisecond + if err := uc.waitForReplicaReadyExpected(context.Background(), tc.checker, "new-primary", 10*time.Millisecond); err == nil { + t.Fatal("unhealthy old primary was accepted") + } + }) + } +} + +func TestUpdateController_ExecuteTargetsRetainsOldPrimaryOnTimeout(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + uc.tickInterval = time.Millisecond + oldPrimary := &testutil.FakeMySQL{ReadOnlyVal: false} + standby := &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "old-primary", + }} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + processed, err := uc.ExecuteTargets(ctx, + UpdateTarget{Name: "old-primary", Checker: oldPrimary, Drifted: true}, + []UpdateTarget{{Name: "standby", Host: "new-primary", Checker: standby, Promotable: true, ExpectedSource: "old-primary"}}, + func(context.Context, string) error { return nil }, nil, + ) + if err == nil { + t.Fatal("expected old-primary readiness failure") + } + if strings.Contains(strings.Join(processed, ","), "old-primary") { + t.Fatalf("old primary drift was cleared: processed=%v", processed) + } +} + func TestUpdateController_IsUpdating(t *testing.T) { logger := testutil.TestLogger() failoverCtl := NewFailoverController(logger) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 532b247..40a531d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -94,6 +94,11 @@ var ( Help: "Whether a replication thread is running (1=yes, 0=no). Thread label is 'io' or 'sql'.", }, []string{"site", "thread"}) + ReplicationSourceState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "bloodraven_replication_source_state", + Help: "Direct-primary replication source convergence as a state-set.", + }, []string{"site", "state"}) + SiteState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "bloodraven_site_state", Help: "Current site state as a state-set (1 for current state, 0 for others). State label is 'writable', 'read-only', 'unreachable', or 'unknown'.", @@ -354,12 +359,15 @@ var ( // AllStates is the set of possible site states, used to emit the full state-set. var AllStates = []string{"writable", "read-only", "unreachable", "unknown"} +// AllSourceStates is the bounded source-convergence state set. +var AllSourceStates = []string{"converged", "pending", "blocked"} + // Register registers all metrics with the given registerer. func Register(reg prometheus.Registerer) { reg.MustRegister(PollLatency, StateTransitions, TaintOperations, WSClientCount, DNSFlipCount, FailoversTotal, PlannedFailoversTotal, PlannedFailoverDurationSeconds, PlannedFailoverLagWaitSeconds, SplitBrainAutoResolveTotal, PrimaryReassertTotal, - ReplicationLag, ReplicationRunning, SiteState, DivergentTransactions, RecloneOperations, + ReplicationLag, ReplicationRunning, ReplicationSourceState, SiteState, DivergentTransactions, RecloneOperations, BackupRunsTotal, BackupDurationSeconds, BackupLastSuccessTimestamp, BackupLastAttemptTimestamp, BackupLastSizeBytes, BackupVerifiedTimestamp, BackupVerificationLastAttemptTimestamp, diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index fddb188..9612da2 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -35,3 +35,20 @@ func TestRegisterIncludesRestoreMetrics(t *testing.T) { } } } + +func TestRegisterIncludesReplicationSourceState(t *testing.T) { + reg := prometheus.NewRegistry() + Register(reg) + ReplicationSourceState.WithLabelValues("reader", "converged").Set(1) + defer ReplicationSourceState.DeleteLabelValues("reader", "converged") + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "bloodraven_replication_source_state" { + return + } + } + t.Fatal("replication source state metric was not registered") +} diff --git a/internal/playground/kube/endpointslices.go b/internal/playground/kube/endpointslices.go new file mode 100644 index 0000000..593f059 --- /dev/null +++ b/internal/playground/kube/endpointslices.go @@ -0,0 +1,106 @@ +package kube + +import ( + "context" + "sort" + + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ServiceEndpoint describes one EndpointSlice endpoint selected by a Service. +type ServiceEndpoint struct { + PodName string + PodUID string + Addresses []string + Ready bool + Serving bool + Terminating bool + PortNames []string +} + +// ServiceEndpointState is the flattened EndpointSlice state for a Service. +type ServiceEndpointState struct { + Ports map[string]int32 + Endpoints []ServiceEndpoint +} + +// ServiceEndpointState reads every EndpointSlice owned by serviceName. +func (c *Client) ServiceEndpointState(ctx context.Context, namespace, serviceName string) (ServiceEndpointState, error) { + if namespace == "" { + namespace = PlaygroundNamespace + } + slices, err := c.Kubernetes.DiscoveryV1().EndpointSlices(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: discoveryv1.LabelServiceName + "=" + serviceName, + }) + if err != nil { + return ServiceEndpointState{}, err + } + + state := ServiceEndpointState{Ports: map[string]int32{}} + for i := range slices.Items { + slice := &slices.Items[i] + var portNames []string + for _, port := range slice.Ports { + if port.Name == nil || port.Port == nil { + continue + } + state.Ports[*port.Name] = *port.Port + portNames = append(portNames, *port.Name) + } + sort.Strings(portNames) + for _, endpoint := range slice.Endpoints { + ready := endpoint.Conditions.Ready != nil && *endpoint.Conditions.Ready + serving := ready + if endpoint.Conditions.Serving != nil { + serving = *endpoint.Conditions.Serving + } + item := ServiceEndpoint{ + Addresses: append([]string(nil), endpoint.Addresses...), + Ready: ready, + Serving: serving, + PortNames: append([]string(nil), portNames...), + } + if endpoint.Conditions.Terminating != nil { + item.Terminating = *endpoint.Conditions.Terminating + } + if endpoint.TargetRef != nil && endpoint.TargetRef.Kind == "Pod" { + item.PodName = endpoint.TargetRef.Name + item.PodUID = string(endpoint.TargetRef.UID) + } + state.Endpoints = append(state.Endpoints, item) + } + } + return state, nil +} + +// ServingPodNames returns sorted Pod targets serving the named port. +func (s ServiceEndpointState) ServingPodNames(portName string) []string { + return s.podNames(portName, func(endpoint ServiceEndpoint) bool { return endpoint.Serving }) +} + +// ReadyPodNames returns sorted Pod targets ready on the named port. +func (s ServiceEndpointState) ReadyPodNames(portName string) []string { + return s.podNames(portName, func(endpoint ServiceEndpoint) bool { return endpoint.Ready }) +} + +func (s ServiceEndpointState) podNames(portName string, include func(ServiceEndpoint) bool) []string { + var names []string + for _, endpoint := range s.Endpoints { + if !include(endpoint) || !containsEndpointPort(endpoint.PortNames, portName) || endpoint.PodName == "" { + continue + } + names = append(names, endpoint.PodName) + } + sort.Strings(names) + return names +} + +func containsEndpointPort(names []string, want string) bool { + for _, name := range names { + if name == want { + return true + } + } + return false +} diff --git a/internal/playground/kube/endpointslices_test.go b/internal/playground/kube/endpointslices_test.go new file mode 100644 index 0000000..277cef6 --- /dev/null +++ b/internal/playground/kube/endpointslices_test.go @@ -0,0 +1,95 @@ +package kube + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" +) + +func TestServiceEndpointStateFlattensSlices(t *testing.T) { + ready, notReady, serving := true, false, true + mysqlName, sidecarName := "mysql", "sidecar" + mysqlPort, sidecarPort := int32(3306), int32(8080) + slice := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reader-abc", + Namespace: "test", + Labels: map[string]string{discoveryv1.LabelServiceName: "mysql-playground-reader"}, + }, + Ports: []discoveryv1.EndpointPort{ + {Name: &mysqlName, Port: &mysqlPort}, + {Name: &sidecarName, Port: &sidecarPort}, + }, + Endpoints: []discoveryv1.Endpoint{ + { + Addresses: []string{"10.0.0.10"}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "reader-ready", UID: types.UID("uid-ready")}, + }, + { + Addresses: []string{"10.0.0.11"}, + Conditions: discoveryv1.EndpointConditions{Ready: ¬Ready, Serving: &serving}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "reader-serving", UID: types.UID("uid-serving")}, + }, + }, + } + c := &Client{Kubernetes: fake.NewSimpleClientset(slice)} + + state, err := c.ServiceEndpointState(context.Background(), "test", "mysql-playground-reader") + if err != nil { + t.Fatalf("ServiceEndpointState() error = %v", err) + } + if got := state.Ports["mysql"]; got != 3306 { + t.Errorf("mysql port = %d, want 3306", got) + } + if got := state.Ports["sidecar"]; got != 8080 { + t.Errorf("sidecar port = %d, want 8080", got) + } + got := state.ServingPodNames("mysql") + want := []string{"reader-ready", "reader-serving"} + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("ServingPodNames(mysql) = %v, want %v", got, want) + } + if got := state.ReadyPodNames("mysql"); len(got) != 1 || got[0] != "reader-ready" { + t.Fatalf("ReadyPodNames(mysql) = %v, want [reader-ready]", got) + } + if got := state.ServingPodNames("missing"); len(got) != 0 { + t.Fatalf("ServingPodNames(missing) = %v, want empty", got) + } +} + +func TestServiceEndpointStateDefaultsServingToReady(t *testing.T) { + ready := false + mysqlName := "mysql" + mysqlPort := int32(3306) + slice := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reader-unready", + Namespace: "test", + Labels: map[string]string{discoveryv1.LabelServiceName: "mysql-playground-reader"}, + }, + Ports: []discoveryv1.EndpointPort{{Name: &mysqlName, Port: &mysqlPort}}, + Endpoints: []discoveryv1.Endpoint{{ + Addresses: []string{"10.0.0.12"}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "reader-unready"}, + }}, + } + c := &Client{Kubernetes: fake.NewSimpleClientset(slice)} + + state, err := c.ServiceEndpointState(context.Background(), "test", "mysql-playground-reader") + if err != nil { + t.Fatalf("ServiceEndpointState() error = %v", err) + } + if got := state.ServingPodNames("mysql"); len(got) != 0 { + t.Fatalf("ServingPodNames(mysql) = %v, want empty", got) + } + if got := state.ReadyPodNames("mysql"); len(got) != 0 { + t.Fatalf("ReadyPodNames(mysql) = %v, want empty", got) + } +} diff --git a/internal/playground/logs/matcher.go b/internal/playground/logs/matcher.go index c3e10aa..b93a407 100644 --- a/internal/playground/logs/matcher.go +++ b/internal/playground/logs/matcher.go @@ -2,7 +2,10 @@ package logs import ( "context" + "encoding/json" + "fmt" "regexp" + "strconv" "strings" "time" ) @@ -33,6 +36,40 @@ func And(preds ...Predicate) Predicate { } } +// Structured matches a structured log message and exact field values in +// either JSON or slog text output. Operator logging format differs between +// local and CI deployments, while the public msg/field contract is stable. +func Structured(msg string, fields map[string]string) Predicate { + return func(line string) bool { + var record map[string]any + if json.Unmarshal([]byte(line), &record) == nil { + if fmt.Sprint(record["msg"]) != msg { + return false + } + for key, value := range fields { + if fmt.Sprint(record[key]) != value { + return false + } + } + return true + } + + if !textFieldMatches(line, "msg", msg) { + return false + } + for key, value := range fields { + if !textFieldMatches(line, key, value) { + return false + } + } + return true + } +} + +func textFieldMatches(line, key, value string) bool { + return strings.Contains(line, key+"="+value) || strings.Contains(line, key+"="+strconv.Quote(value)) +} + // Wait blocks until a line matching the predicate is observed in the // tailer's ring buffer, or the context expires. The search begins at // the supplied since time — callers pass time.Now() before injecting diff --git a/internal/playground/logs/matcher_test.go b/internal/playground/logs/matcher_test.go index 2d13f90..678b3bb 100644 --- a/internal/playground/logs/matcher_test.go +++ b/internal/playground/logs/matcher_test.go @@ -44,3 +44,21 @@ func TestTailerWaitTimeoutReturnsCtxErr(t *testing.T) { t.Fatalf("expected timeout error, got nil") } } + +func TestStructuredMatchesJSONAndText(t *testing.T) { + pred := Structured("starting bootstrap", map[string]string{ + "source": "auto-clone", + "recipient": "reader", + }) + for _, line := range []string{ + `{"msg":"starting bootstrap","source":"auto-clone","recipient":"reader"}`, + `time=now level=INFO msg="starting bootstrap" source=auto-clone recipient=reader`, + } { + if !pred(line) { + t.Errorf("Structured predicate did not match %q", line) + } + } + if pred(`{"msg":"starting bootstrap","source":"reclone","recipient":"reader"}`) { + t.Error("Structured predicate matched the wrong source") + } +} diff --git a/internal/playground/runner/executor.go b/internal/playground/runner/executor.go index bf4ae93..8e8d1be 100644 --- a/internal/playground/runner/executor.go +++ b/internal/playground/runner/executor.go @@ -219,7 +219,7 @@ func (e *Executor) cleanup(env *Env, s Scenario) error { if err := env.Chaos.GlobalRecover(cleanCtx); err != nil { errs = append(errs, fmt.Errorf("global recover: %w", err)) } - // Wait for the cluster to converge back to a healthy two-site + // Wait for the cluster to converge back to a healthy N-site // state so the next scenario's Precheck does not race a still- // recovering cluster (run-all surfaced this: scenario N's chaos // scales a site to 0; scenario N+1 starts before that pod has @@ -310,7 +310,7 @@ func processAlive(pid int) bool { return p.Signal(nil) == nil } -// waitForClusterReconverge polls the MFG until both sites are in +// waitForClusterReconverge polls the MFG until every site is in // {writable, read-only} (i.e. the operator has finished any in-flight // promotion/recovery), the cluster reports Ready=True, and optional // Dragonfly topology has returned to Ready. This is the cleanup gate diff --git a/internal/playground/runner/profile.go b/internal/playground/runner/profile.go index 075e7e1..11b3186 100644 --- a/internal/playground/runner/profile.go +++ b/internal/playground/runner/profile.go @@ -9,7 +9,8 @@ package runner // behaviours (emergency failover, planned switchover, operator restart, // data integrity, operator kill during failover, self-fencing, // network partition, PVC loss/re-bootstrap, old-primary recovery, -// failover state durability, backup verification, PITR verification). +// failover state durability, backup verification, PITR verification, +// reader PVC loss, and direct-source auto-clone). // - full: every registered scenario (existing run-all behaviour). type Profile string @@ -54,6 +55,7 @@ var releaseScenarios = map[string]bool{ "23-failover-state-durability": true, // state survives operator restart "30-backup-verification-rustfs": true, // RustFS backup restore verification "31-pitr-verification-rustfs": true, // RustFS PITR replay verification + "40-reader-data-loss-reclone": true, // reader PVC loss, endpoint shedding, and direct-source auto-clone } // Profiles returns the list of valid profile names for CLI help and diff --git a/internal/playground/runner/profile_registry_test.go b/internal/playground/runner/profile_registry_test.go index f57b779..d84c5a6 100644 --- a/internal/playground/runner/profile_registry_test.go +++ b/internal/playground/runner/profile_registry_test.go @@ -18,13 +18,13 @@ func TestProfilesSelectRegisteredScenarios(t *testing.T) { t.Fatalf("smoke profile selected %d scenarios, want 3", len(smoke)) } - // The release subset lists 12 members, all of which are now selected: + // The release subset lists 13 members, all of which are now selected: // 31-pitr-verification-rustfs is no longer quarantined (#101 fixed — the // verify mysqld runs gtid_mode=ON and server-side dedup handles the PITR // replay). release := runner.SelectForProfile(all, runner.ProfileRelease) - if len(release) != 12 { - t.Fatalf("release profile selected %d scenarios, want 12", len(release)) + if len(release) != 13 { + t.Fatalf("release profile selected %d scenarios, want 13", len(release)) } if !containsScenarioID(release, "09-network-partition-self-fence") { t.Error("release profile must include 09-network-partition-self-fence (no longer quarantined)") @@ -32,6 +32,9 @@ func TestProfilesSelectRegisteredScenarios(t *testing.T) { if !containsScenarioID(release, "31-pitr-verification-rustfs") { t.Error("31-pitr-verification-rustfs is no longer quarantined (#101) and must be in the release profile") } + if !containsScenarioID(release, "40-reader-data-loss-reclone") { + t.Error("40-reader-data-loss-reclone must be in the release profile") + } // full = every registered scenario minus any that are quarantined. Computed // from the live quarantine state so this stays correct regardless of how diff --git a/internal/playground/scenarios/common.go b/internal/playground/scenarios/common.go index fb4f248..8d24011 100644 --- a/internal/playground/scenarios/common.go +++ b/internal/playground/scenarios/common.go @@ -104,13 +104,13 @@ func CheckBaseline(ctx context.Context, k *pgkube.Client, namespace, fg string) // digging through operator logs for a real fault when the // fix is just to wipe stale status. if bothReadOnly(mfg) { - return fmt.Errorf("baseline unhealthy: operator sees both sites read-only and refuses to auto-promote (matrix.go startup-state guard) — run ./playground/reset-mysql.sh") + return fmt.Errorf("baseline unhealthy: operator sees all sites read-only and refuses to auto-promote (matrix.go startup-state guard) — run ./playground/reset-mysql.sh") } return fmt.Errorf("baseline unhealthy: Ready condition is %q (run ./playground/setup.sh)", pgkube.ReadyCondition(mfg)) } if mfg.Status.ActiveSite == "" { if bothReadOnly(mfg) { - return fmt.Errorf("baseline unhealthy: operator sees both sites read-only and refuses to auto-promote (matrix.go startup-state guard) — run ./playground/reset-mysql.sh") + return fmt.Errorf("baseline unhealthy: operator sees all sites read-only and refuses to auto-promote (matrix.go startup-state guard) — run ./playground/reset-mysql.sh") } return fmt.Errorf("baseline unhealthy: no active site (run ./playground/setup.sh)") } diff --git a/internal/playground/scenarios/inventory_40_test.go b/internal/playground/scenarios/inventory_40_test.go new file mode 100644 index 0000000..6728167 --- /dev/null +++ b/internal/playground/scenarios/inventory_40_test.go @@ -0,0 +1,70 @@ +package scenarios + +import ( + "strings" + "testing" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +func TestScenario40RegisteredAndReleaseProfiled(t *testing.T) { + const id = "40-reader-data-loss-reclone" + scenario, ok := runner.DefaultRegistry.Get(id) + if !ok { + t.Fatalf("scenario %s is not registered", id) + } + if scenario.Quarantine != "" { + t.Fatalf("scenario %s must not be quarantined: %q", id, scenario.Quarantine) + } + if !scenario.ResetBeforeRunAll { + t.Fatalf("scenario %s must reset before run-all", id) + } + if scenario.Timeout <= 0 || scenario.Precheck == nil || len(scenario.Steps) == 0 { + t.Fatalf("scenario %s is incomplete: timeout=%s precheck=%v steps=%d", id, scenario.Timeout, scenario.Precheck != nil, len(scenario.Steps)) + } + if !inventoryContains(runner.SelectForProfile(runner.DefaultRegistry.List(), runner.ProfileRelease), id) { + t.Fatalf("scenario %s is not selected by the release profile", id) + } +} + +func TestCanonicalMySQLHost(t *testing.T) { + for _, input := range []string{ + "MYSQL-PLAYGROUND-IAD-INTERNAL.NS.SVC.CLUSTER.LOCAL.", + " mysql-playground-iad-internal.ns.svc.cluster.local:3306 ", + } { + if got, want := canonicalMySQLHost(input), "mysql-playground-iad-internal.ns.svc.cluster.local"; got != want { + t.Errorf("canonicalMySQLHost(%q) = %q, want %q", input, got, want) + } + } +} + +func TestS40AssertReaderStatus(t *testing.T) { + maxLag := int64(10) + lag := int64(10) + mfg := &v1alpha1.MysqlFailoverGroup{ + Spec: v1alpha1.MysqlFailoverGroupSpec{ + Replication: &v1alpha1.ReplicationSpec{ReadOnlyMaxLagSeconds: &maxLag}, + }, + } + status := v1alpha1.SiteStatus{ + Name: "reader", + State: "read-only", + Replicating: true, + SecondsBehindSource: &lag, + SourceHost: "mysql-playground-iad-internal.ns.svc.cluster.local:3306", + SourceConvergenceState: v1alpha1.SourceConvergenceConverged, + } + if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err != nil { + t.Fatalf("exact-threshold reader rejected: %v", err) + } + + lag = 11 + if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("over-threshold error = %v, want exceeds", err) + } + status.SecondsBehindSource = nil + if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown-lag error = %v, want unknown", err) + } +} diff --git a/internal/playground/scenarios/s02_planned_switchover.go b/internal/playground/scenarios/s02_planned_switchover.go index 215612a..c5eaf71 100644 --- a/internal/playground/scenarios/s02_planned_switchover.go +++ b/internal/playground/scenarios/s02_planned_switchover.go @@ -3,10 +3,12 @@ package scenarios import ( "context" "fmt" + "sort" "time" v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" pgmetrics "github.com/shipstream/bloodraven/internal/playground/metrics" + pgmysql "github.com/shipstream/bloodraven/internal/playground/mysql" "github.com/shipstream/bloodraven/internal/playground/runner" ) @@ -31,12 +33,78 @@ func scenario02PlannedSwitchover() runner.Scenario { Steps: []runner.Step{ injectPlannedFailoverAnnotation(), observePlannedFailoverSucceeded(), + verifyFollowersDirectlyFollowNewPrimary(), verifyTransactionsLostZero(), verifyPlannedFailoverMetric(), }, } } +func verifyFollowersDirectlyFollowNewPrimary() runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "every follower directly replicates from the new active primary", + Do: func(ctx context.Context, env *runner.Env) error { + target := ctxFetch(env, "switchoverTarget") + original := ctxFetch(env, "originalPrimary") + expectedHost := playgroundInternalSiteHost(env.FG, target, env.Namespace) + deadline := time.Now().Add(2 * time.Minute) + var last string + for time.Now().Before(deadline) { + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + last = err.Error() + } else if mfg.Status.ActiveSite != target { + return fmt.Errorf("planned failover target changed after success: activeSite=%q want %q", mfg.Status.ActiveSite, target) + } else { + var bad []string + foundOriginal := false + foundReader := false + for _, site := range mfg.Spec.Sites { + if site.Name == target { + continue + } + foundOriginal = foundOriginal || site.Name == original + foundReader = foundReader || site.IsReadOnlyReader() + client, openErr := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, site.Name, env.Creds) + if openErr != nil { + bad = append(bad, fmt.Sprintf("%s=open:%v", site.Name, openErr)) + continue + } + rs, statusErr := client.ShowReplicaStatus(ctx) + _ = client.Close() + if statusErr != nil { + bad = append(bad, fmt.Sprintf("%s=status:%v", site.Name, statusErr)) + continue + } + if !rs.Configured || !rs.IORunning || !rs.SQLRunning || canonicalMySQLHost(rs.SourceHost) != canonicalMySQLHost(expectedHost) { + bad = append(bad, fmt.Sprintf("%s=configured:%v/io:%v/sql:%v/source:%q", site.Name, rs.Configured, rs.IORunning, rs.SQLRunning, rs.SourceHost)) + } + } + sort.Strings(bad) + if !foundOriginal { + return fmt.Errorf("demoted original primary %q is not a follower in spec", original) + } + if !foundReader { + return fmt.Errorf("planned switchover topology has no read-only reader follower") + } + last = fmt.Sprintf("expectedSource=%s bad=%v", expectedHost, bad) + if len(bad) == 0 { + env.Capture.Note("all followers directly replicate from " + expectedHost) + return nil + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("followers did not converge directly to %s within 2m (last: %s)", expectedHost, last) + }, + } +} + func injectPlannedFailoverAnnotation() runner.Step { return runner.Step{ Phase: runner.PhaseInject, diff --git a/internal/playground/scenarios/s05_operator_kill_during_failover.go b/internal/playground/scenarios/s05_operator_kill_during_failover.go index 5ba569c..377e3d7 100644 --- a/internal/playground/scenarios/s05_operator_kill_during_failover.go +++ b/internal/playground/scenarios/s05_operator_kill_during_failover.go @@ -170,7 +170,7 @@ func s05ObserveSafeConvergence() runner.Step { defer convergeCancel() var healthySince time.Time _, err := env.Wait.UntilCR(convergeCtx, env.Namespace, - "writable=1 read-only=1 ready=True blocked=0 (stable for 10s)", + "writable=1 read-only=N-1 ready=True blocked=0 (stable for 10s)", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked []string for _, s := range mfg.Status.Sites { @@ -196,7 +196,7 @@ func s05ObserveSafeConvergence() runner.Step { } } healthy := mfg.Status.ActiveSite != "" && - len(writable) == 1 && len(readOnly) == 1 && + len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && ready == "True" if healthy { if healthySince.IsZero() { diff --git a/internal/playground/scenarios/s06_self_fence_isolated_primary.go b/internal/playground/scenarios/s06_self_fence_isolated_primary.go index 185208d..127cf10 100644 --- a/internal/playground/scenarios/s06_self_fence_isolated_primary.go +++ b/internal/playground/scenarios/s06_self_fence_isolated_primary.go @@ -14,7 +14,7 @@ func init() { runner.Register(scenario06SelfFenceIsolatedPrimary()) } -// scenario06SelfFenceIsolatedPrimary scales the operator and the peer +// scenario06SelfFenceIsolatedPrimary scales the operator and every peer // MySQL site to 0 replicas, leaving the active primary unable to talk // to either Bloodraven or any peer sidecar. After leaseTimeout (20s in // the playground) the primary's sidecar must self-fence — set @@ -30,8 +30,8 @@ func init() { func scenario06SelfFenceIsolatedPrimary() runner.Scenario { return runner.Scenario{ ID: "06-self-fence-isolated-primary", - Title: "Isolated primary self-fences when operator and peer are gone", - Hypothesis: "With the operator scaled to 0 AND the peer site scaled to 0, the active primary's " + + Title: "Isolated primary self-fences when operator and all peers are gone", + Hypothesis: "With the operator scaled to 0 AND every peer site scaled to 0, the active primary's " + "sidecar exceeds leaseTimeout with no reachable Bloodraven and no reachable peer, sets " + "super_read_only=ON, and logs SELF-FENCED.", Risk: "medium", @@ -50,24 +50,21 @@ func scenario06SelfFenceIsolatedPrimary() runner.Scenario { func injectIsolatePrimary() runner.Step { return runner.Step{ Phase: runner.PhaseInject, - Name: "open tailers, scale operator and peer to 0", + Name: "open tailers, scale operator and every non-self peer to 0", Do: func(ctx context.Context, env *runner.Env) error { mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) if err != nil { return err } active := mfg.Status.ActiveSite - peer, err := PeerOf(mfg, active) - if err != nil { - return err + peers := mfg.Spec.PeerSiteNames(active) + if len(peers) == 0 { + return fmt.Errorf("active site %s has no peers", active) } - env.Capture.Note(fmt.Sprintf("active=%s peer=%s; isolating primary by killing operator+peer", active, peer)) + env.Capture.Note(fmt.Sprintf("active=%s peers=%v; isolating primary by scaling operator+all peers to zero", active, peers)) if err := ctxStash(ctx, env, "primarySite", active); err != nil { return err } - if err := ctxStash(ctx, env, "peerSite", peer); err != nil { - return err - } // Open tailer + sidecar probe BEFORE injection. The primary // pod stays running, so the port-forward survives — but // opening it after isolation would race the sidecar's @@ -78,13 +75,15 @@ func injectIsolatePrimary() runner.Step { if _, err := env.Sidecar(active); err != nil { return fmt.Errorf("open sidecar probe for %s: %w", active, err) } - // Order matters: kill the peer first so the primary's + // Order matters: kill all peers first so the primary's // peer-check timer starts ticking, then kill the operator. // Reverters are LIFO, so cleanup brings the operator back // before the peer — closer to a normal recovery shape than // the reverse. - if err := env.Chaos.ScaleSiteToZero(ctx, peer); err != nil { - return err + for _, peer := range peers { + if err := env.Chaos.ScaleSiteToZero(ctx, peer); err != nil { + return fmt.Errorf("scale peer %s to zero: %w", peer, err) + } } if err := env.Chaos.ScaleOperatorToZero(ctx); err != nil { return err diff --git a/internal/playground/scenarios/s08_gtid_divergence_detection.go b/internal/playground/scenarios/s08_gtid_divergence_detection.go index 066a753..2d89760 100644 --- a/internal/playground/scenarios/s08_gtid_divergence_detection.go +++ b/internal/playground/scenarios/s08_gtid_divergence_detection.go @@ -164,7 +164,7 @@ func s08AutoRecloneCleanup(ctx context.Context, env *runner.Env) error { div = append(div, s.Name) } } - done := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 && len(div) == 0 + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && len(div) == 0 return done, fmt.Sprintf("writable=%v read-only=%v blocked=%v divergent=%v", writable, readOnly, blocked, div), nil }, diff --git a/internal/playground/scenarios/s10_full_bootstrap_after_data_wipe.go b/internal/playground/scenarios/s10_full_bootstrap_after_data_wipe.go index c5bf5b8..4a7e66d 100644 --- a/internal/playground/scenarios/s10_full_bootstrap_after_data_wipe.go +++ b/internal/playground/scenarios/s10_full_bootstrap_after_data_wipe.go @@ -174,7 +174,7 @@ func s10ObserveClusterReconverged() runner.Step { waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "sites: writable=1 read-only=1 divergent=0 blocked=0", + "sites: writable=1 read-only=N-1 divergent=0 blocked=0", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked, divergent []string for _, s := range mfg.Status.Sites { @@ -197,7 +197,7 @@ func s10ObserveClusterReconverged() runner.Step { sort.Strings(readOnly) msg := fmt.Sprintf("writable=%v read-only=%v other=%v blocked=%v divergent=%v", writable, readOnly, other, blocked, divergent) - done := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 && len(divergent) == 0 + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && len(divergent) == 0 return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s11_total_loss_recovery.go b/internal/playground/scenarios/s11_total_loss_recovery.go index a5539a4..826b7e2 100644 --- a/internal/playground/scenarios/s11_total_loss_recovery.go +++ b/internal/playground/scenarios/s11_total_loss_recovery.go @@ -19,8 +19,8 @@ func init() { // playground's only TOTAL-LOSS path — and asserts the operator // recognises and surfaces it (`ALERT` log with `TOTAL LOSS: all sites // are unreachable`) without crashing or wedging the reconciler. It -// then scales both back up and asserts the cluster reconverges to a -// single writable + single read-only site. +// then scales every site back up and asserts the cluster reconverges to a +// single writable site with every follower read-only. // // The operator's resilience here is the property under test: a TOTAL // LOSS event must not put the controller into a state from which the @@ -30,10 +30,10 @@ func init() { func scenario11TotalLossRecovery() runner.Scenario { return runner.Scenario{ ID: "11-total-loss-recovery", - Title: "Both sites scaled to 0 — TOTAL LOSS surfaced and recovered", + Title: "Every site scaled to 0 — TOTAL LOSS surfaced and recovered", Hypothesis: "When every MySQL site is unreachable simultaneously, the operator emits an `ALERT` " + - "log line containing `TOTAL LOSS: all sites are unreachable` and does NOT crash. After both " + - "sites are restored, the cluster reconverges to one writable + one read-only with no " + + "log line containing `TOTAL LOSS: all sites are unreachable` and does NOT crash. After every " + + "site is restored, the cluster reconverges to one writable with all followers read-only and no " + "RecoveryBlocked sites.", Risk: "high", DocLink: "playground/chaos-scenarios.md#11-simultaneous-both-site-kill", @@ -126,7 +126,7 @@ func injectScaleAllSitesBackUp() runner.Step { func observeTotalLossReconvergence() runner.Step { return runner.Step{ Phase: runner.PhaseObserve, - Name: "cluster reconverges to one writable + one read-only after TOTAL LOSS", + Name: "cluster reconverges to one writable + all followers read-only after TOTAL LOSS", Do: func(ctx context.Context, env *runner.Env) error { // 4 minutes: bringing two MySQL pods back up, restarting the // replication topology, and re-establishing replication can @@ -136,7 +136,7 @@ func observeTotalLossReconvergence() runner.Step { waitCtx, cancel := context.WithTimeout(ctx, 4*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "sites: writable=1 read-only=1 blocked=0 active!=\"\"", + "sites: writable=1 read-only=N-1 blocked=0 active!=\"\"", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked []string for _, s := range mfg.Status.Sites { @@ -156,7 +156,7 @@ func observeTotalLossReconvergence() runner.Step { sort.Strings(readOnly) msg := fmt.Sprintf("active=%q writable=%v read-only=%v other=%v blocked=%v", mfg.Status.ActiveSite, writable, readOnly, other, blocked) - done := mfg.Status.ActiveSite != "" && len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 + done := mfg.Status.ActiveSite != "" && len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s12_old_primary_recovery.go b/internal/playground/scenarios/s12_old_primary_recovery.go index f93a758..8970780 100644 --- a/internal/playground/scenarios/s12_old_primary_recovery.go +++ b/internal/playground/scenarios/s12_old_primary_recovery.go @@ -125,7 +125,7 @@ func observeClusterReconvergence() runner.Step { waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "sites: writable=1 read-only=1 divergent=0", + "sites: writable=1 read-only=N-1 divergent=0", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other []string var divergent int64 @@ -152,7 +152,7 @@ func observeClusterReconvergence() runner.Step { "writable=%v read-only=%v other=%v divergent=%d blocked=%v", writable, readOnly, other, divergent, blocked, ) - done := len(writable) == 1 && len(readOnly) == 1 && divergent == 0 && len(blocked) == 0 + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && divergent == 0 && len(blocked) == 0 return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s12_rolling_update_healthy_state.go b/internal/playground/scenarios/s12_rolling_update_healthy_state.go index c18b3df..3fbf4ff 100644 --- a/internal/playground/scenarios/s12_rolling_update_healthy_state.go +++ b/internal/playground/scenarios/s12_rolling_update_healthy_state.go @@ -26,8 +26,8 @@ const ( // request and asserts the operator's UpdateController drives an ordered // roll: the standby rolls first (UpdateReplica → WaitReplica), the // operator fails over to it (Failover), then the old primary rolls -// (UpdateOldPrimary → WaitOldPrimary → Complete). At the end both pods -// run with the new memory request, both sites are in {writable, +// (UpdateOldPrimary → WaitOldPrimary → Complete). At the end every pod +// runs with the new memory request, every site is in {writable, // read-only}, and no TOTAL LOSS log line was emitted during the window. // // The assertion is the *invariant*, not the exact phase sequence: @@ -47,7 +47,7 @@ func scenario12RollingUpdateHealthyState() runner.Scenario { Title: "Rolling update during healthy state", Hypothesis: "Patching spec.sites[*].resources.requests.memory triggers the UpdateController: " + "status.updatePhase visits at least one non-empty phase, returns to empty after completion, " + - "both deployments end at the new memory request, and no TOTAL LOSS log fires during the roll.", + "every deployment ends at the new memory request, and no TOTAL LOSS log fires during the roll.", Risk: "medium", DocLink: "playground/chaos-scenarios.md#12-rolling-update-during-healthy-state", Timeout: 10 * time.Minute, @@ -107,19 +107,19 @@ func s12ObserveUpdateStarted() runner.Step { } // s12ObserveUpdateComplete waits for the cluster to settle: updatePhase -// returns to empty AND the cluster is in {1 writable, 1 read-only}. The +// returns to empty AND the cluster is in {1 writable, N-1 read-only}. The // updater clears the phase on its own when the roll lands successfully; // if it gets stuck in "Complete" or any non-empty value past 5 minutes // we want to know. func s12ObserveUpdateComplete() runner.Step { return runner.Step{ Phase: runner.PhaseObserve, - Name: "status.updatePhase returns to empty and sites are 1 writable + 1 read-only", + Name: "status.updatePhase returns to empty and sites are 1 writable + all followers read-only", Do: func(ctx context.Context, env *runner.Env) error { waitCtx, cancel := context.WithTimeout(ctx, 8*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "updatePhase=\"\" writable=1 read-only=1", + "updatePhase=\"\" writable=1 read-only=N-1", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other []string for _, s := range mfg.Status.Sites { @@ -136,7 +136,7 @@ func s12ObserveUpdateComplete() runner.Step { sort.Strings(readOnly) msg := fmt.Sprintf("updatePhase=%q writable=%v read-only=%v other=%v", mfg.Status.UpdatePhase, writable, readOnly, other) - done := mfg.Status.UpdatePhase == "" && len(writable) == 1 && len(readOnly) == 1 + done := mfg.Status.UpdatePhase == "" && len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 return done, msg, nil }, ) @@ -145,7 +145,7 @@ func s12ObserveUpdateComplete() runner.Step { } } -// s12VerifyDeploymentsRolled confirms both site deployments now run +// s12VerifyDeploymentsRolled confirms every site deployment now runs // with the patched memory request. We compare quantities (not raw // strings) so "300Mi" and "300M" differences would pass — the resource // API canonicalizes the value. diff --git a/internal/playground/scenarios/s13_operator_kill_during_bootstrap.go b/internal/playground/scenarios/s13_operator_kill_during_bootstrap.go index 61921db..d4cd4d1 100644 --- a/internal/playground/scenarios/s13_operator_kill_during_bootstrap.go +++ b/internal/playground/scenarios/s13_operator_kill_during_bootstrap.go @@ -197,7 +197,7 @@ func s13ObserveClusterReconverged() runner.Step { waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "sites: writable=1 read-only=1 divergent=0 blocked=0", + "sites: writable=1 read-only=N-1 divergent=0 blocked=0", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked, divergent []string for _, s := range mfg.Status.Sites { @@ -220,7 +220,7 @@ func s13ObserveClusterReconverged() runner.Step { sort.Strings(readOnly) msg := fmt.Sprintf("writable=%v read-only=%v other=%v blocked=%v divergent=%v", writable, readOnly, other, blocked, divergent) - done := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 && len(divergent) == 0 + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && len(divergent) == 0 return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s16_mysql_process_kill.go b/internal/playground/scenarios/s16_mysql_process_kill.go index fde0357..03e21ad 100644 --- a/internal/playground/scenarios/s16_mysql_process_kill.go +++ b/internal/playground/scenarios/s16_mysql_process_kill.go @@ -180,7 +180,7 @@ func s16ObserveContainerRestarted() runner.Step { func s16VerifyNoSplitBrain() runner.Step { return runner.Step{ Phase: runner.PhaseVerify, - Name: "cluster converges to exactly one writable + one read-only after SHUTDOWN", + Name: "cluster converges to exactly one writable + all followers read-only after SHUTDOWN", Do: func(ctx context.Context, env *runner.Env) error { deadline := time.Now().Add(3 * time.Minute) tick := time.NewTicker(2 * time.Second) @@ -208,14 +208,14 @@ func s16VerifyNoSplitBrain() runner.Step { if len(writable) > 1 { return fmt.Errorf("split-brain observed after mysqld SHUTDOWN: writable=%v", writable) } - if len(writable) == 1 && len(readOnly) == 1 { + if len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 { env.Capture.Note(fmt.Sprintf("converged: writable=%v read-only=%v", writable, readOnly)) return nil } } if time.Now().After(deadline) { last, lerr := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) - return fmt.Errorf("cluster did not converge to 1 writable + 1 read-only within 3m (last fetch err=%v sites=%+v)", + return fmt.Errorf("cluster did not converge to 1 writable + N-1 read-only within 3m (last fetch err=%v sites=%+v)", lerr, summarizeSites(last)) } select { diff --git a/internal/playground/scenarios/s18_rapid_cr_spec_changes.go b/internal/playground/scenarios/s18_rapid_cr_spec_changes.go index 776b928..9133f85 100644 --- a/internal/playground/scenarios/s18_rapid_cr_spec_changes.go +++ b/internal/playground/scenarios/s18_rapid_cr_spec_changes.go @@ -35,7 +35,7 @@ const ( // runtime's retry behavior may dedupe them). // // The end-state invariants are the assertion: -// - exactly one site has state=writable, exactly one has state=read-only +// - exactly one site has state=writable, every follower has state=read-only // - no sites are RecoveryBlocked // - the final memory request matches the LAST patch we applied // (s18PatchBaseMi + (s18PatchCount-1)*10) @@ -47,7 +47,7 @@ func scenario18RapidCRSpecChanges() runner.Scenario { ID: "18-rapid-cr-spec-changes-during-failover", Title: "Rapid CR spec changes during failover converge cleanly", Hypothesis: "Triggering a failover and immediately applying five rapid memory-request patches " + - "yields a clean converged state: exactly one writable site, exactly one read-only site, no " + + "yields a clean converged state: exactly one writable site, every follower read-only, no " + "RecoveryBlocked, and final memory request matches the last patch applied.", Risk: "high", DocLink: "playground/chaos-scenarios.md#18-rapid-cr-spec-changes-during-active-failover", @@ -173,18 +173,18 @@ func s18ScaleOldPrimaryBackUp() runner.Step { } // s18ObserveConvergence waits for the cluster to settle: exactly one -// writable site, exactly one read-only site, no RecoveryBlocked, and +// writable site, every follower read-only, no RecoveryBlocked, and // status.updatePhase empty (the rolling-update controller may engage // during the patch storm to roll the new memory request). func s18ObserveConvergence() runner.Step { return runner.Step{ Phase: runner.PhaseObserve, - Name: "cluster converges to {1 writable, 1 read-only} with updatePhase empty", + Name: "cluster converges to {1 writable, N-1 read-only} with updatePhase empty", Do: func(ctx context.Context, env *runner.Env) error { waitCtx, cancel := context.WithTimeout(ctx, 8*time.Minute) defer cancel() _, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "writable=1 read-only=1 blocked=0 updatePhase=\"\"", + "writable=1 read-only=N-1 blocked=0 updatePhase=\"\"", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked []string for _, s := range mfg.Status.Sites { @@ -204,7 +204,7 @@ func s18ObserveConvergence() runner.Step { sort.Strings(readOnly) msg := fmt.Sprintf("writable=%v read-only=%v other=%v blocked=%v updatePhase=%q", writable, readOnly, other, blocked, mfg.Status.UpdatePhase) - done := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 && mfg.Status.UpdatePhase == "" + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && mfg.Status.UpdatePhase == "" return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s19_reclone_interlock.go b/internal/playground/scenarios/s19_reclone_interlock.go index d20fb64..1e5aee8 100644 --- a/internal/playground/scenarios/s19_reclone_interlock.go +++ b/internal/playground/scenarios/s19_reclone_interlock.go @@ -381,7 +381,7 @@ func verifyRecloneAcceptedMatching() runner.Step { } bootDone = boot.Status == metav1.ConditionFalse && boot.Reason == "Done" } - clean := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 && len(divergent) == 0 && bootDone + clean := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 && len(divergent) == 0 && bootDone now := time.Now() if clean { if cleanSince.IsZero() { diff --git a/internal/playground/scenarios/s22_replication_status_after_recovery.go b/internal/playground/scenarios/s22_replication_status_after_recovery.go index 20e20f1..c4e041d 100644 --- a/internal/playground/scenarios/s22_replication_status_after_recovery.go +++ b/internal/playground/scenarios/s22_replication_status_after_recovery.go @@ -101,8 +101,8 @@ func s22ObserveFailoverFlip() runner.Step { } // s22ObserveRecovery scales the original primary back up and waits -// for the cluster to reach a healthy two-site state — exactly one -// writable, exactly one read-only, no RecoveryBlocked. We do not +// for the cluster to reach a healthy N-site state — exactly one +// writable, every follower read-only, no RecoveryBlocked. We do not // inspect the .replicating field here; that's the next step's job. // The point of this step is to land at the moment the operator // considers the recovery finished, so we can time the 30s window @@ -115,11 +115,11 @@ func s22ObserveRecovery() runner.Step { if err := env.Chaos.Revert(ctx); err != nil { return fmt.Errorf("scale old primary back up: %w", err) } - env.Capture.Note("old primary scaled back to 1; awaiting healthy two-site convergence") + env.Capture.Note("old primary scaled back to 1; awaiting healthy N-site convergence") waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() mfg, err := env.Wait.UntilCR(waitCtx, env.Namespace, - "sites: writable=1 read-only=1 blocked=0", + "sites: writable=1 read-only=N-1 blocked=0", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { var writable, readOnly, other, blocked []string for _, s := range mfg.Status.Sites { @@ -137,7 +137,7 @@ func s22ObserveRecovery() runner.Step { } msg := fmt.Sprintf("writable=%v read-only=%v other=%v blocked=%v", writable, readOnly, other, blocked) - done := len(writable) == 1 && len(readOnly) == 1 && len(blocked) == 0 + done := len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && len(blocked) == 0 return done, msg, nil }, ) diff --git a/internal/playground/scenarios/s34_operator_kill_during_wait_replica.go b/internal/playground/scenarios/s34_operator_kill_during_wait_replica.go index 5833dcd..3af10c3 100644 --- a/internal/playground/scenarios/s34_operator_kill_during_wait_replica.go +++ b/internal/playground/scenarios/s34_operator_kill_during_wait_replica.go @@ -34,17 +34,17 @@ const ( // fails), force-deletes the operator pod, then removes the hold. The // replacement operator — whose in-memory update phase was lost — must // re-derive the remaining drift from the Deployment spec-hash mismatch and -// finish the roll: both deployments end at the new memory request, no double -// -roll or TOTAL LOSS occurs, and the cluster settles at one writable + one -// read-only. +// finish the roll: every deployment ends at the new memory request, no double +// -roll or TOTAL LOSS occurs, and the cluster settles at one writable with +// every follower read-only. func scenario34OperatorKillDuringWaitReplica() runner.Scenario { return runner.Scenario{ ID: s34ID, Title: "Operator kill during ordered-update WaitReplica re-derives and completes", - Hypothesis: "Killing the operator while an ordered update is held at WaitReplica does not double-roll both " + + Hypothesis: "Killing the operator while an ordered update is held at WaitReplica does not double-roll all " + "MySQL pods or cause TOTAL LOSS. The replacement operator re-derives remaining drift from Deployment " + - "spec-hash mismatch and completes: status.updatePhase returns to empty, both deployments run the new " + - "memory request, and the cluster is one writable + one read-only.", + "spec-hash mismatch and completes: status.updatePhase returns to empty, every deployment runs the new " + + "memory request, and the cluster is one writable with all followers read-only.", Risk: "high", DocLink: "playground/chaos-scenarios.md#34-operator-kill-during-ordered-update-waitreplica", Timeout: 12 * time.Minute, @@ -189,7 +189,7 @@ func s34ObserveReplacementCompletes() runner.Step { sort.Strings(readOnly) depOK, depMsg := s34DeploymentsAt(doneCtx, env, mfg, expected) msg := fmt.Sprintf("updatePhase=%q writable=%v read-only=%v other=%v deployments=%s", mfg.Status.UpdatePhase, writable, readOnly, other, depMsg) - done := mfg.Status.UpdatePhase == "" && len(writable) == 1 && len(readOnly) == 1 && depOK + done := mfg.Status.UpdatePhase == "" && len(writable) == 1 && len(readOnly) == len(mfg.Status.Sites)-1 && depOK return done, msg, nil }) return err @@ -197,7 +197,7 @@ func s34ObserveReplacementCompletes() runner.Step { } } -// s34DeploymentsAt reports whether both site deployments run at the wanted +// s34DeploymentsAt reports whether every site deployment runs at the wanted // memory request. Returns a message for progress logging. Takes the caller's // context so the polling predicate stops issuing API calls when the step's // wait is cancelled or times out. diff --git a/internal/playground/scenarios/s40_observer_test.go b/internal/playground/scenarios/s40_observer_test.go new file mode 100644 index 0000000..583879c --- /dev/null +++ b/internal/playground/scenarios/s40_observer_test.go @@ -0,0 +1,200 @@ +package scenarios + +import ( + "context" + "errors" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +func TestS40ObserveOnceFailsMFGReadError(t *testing.T) { + wantErr := errors.New("MFG API unavailable") + env, state := s40ObserverTestEnv(t, k8sfake.NewSimpleClientset()) + underlying, ok := env.Kube.Controller.(client.WithWatch) + if !ok { + t.Fatal("fake controller client does not implement client.WithWatch") + } + env.Kube.Controller = interceptor.NewClient(underlying, interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return wantErr + }, + }) + + observation := &s40ContinuousObservation{} + s40ObserveOnce(context.Background(), env, state, observation) + s40RequireObservationError(t, observation, "read MFG", wantErr.Error()) +} + +func TestS40ObserveOnceFailsClientEndpointSliceReadError(t *testing.T) { + wantErr := apierrors.NewNotFound(schema.GroupResource{Group: discoveryv1.GroupName, Resource: "endpointslices"}, "reader-client") + clientset := k8sfake.NewSimpleClientset() + clientset.PrependReactor("list", "endpointslices", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, wantErr + }) + env, state := s40ObserverTestEnv(t, clientset) + + observation := &s40ContinuousObservation{} + s40ObserveOnce(context.Background(), env, state, observation) + s40RequireObservationError(t, observation, "read reader client EndpointSlices", "not found") +} + +func TestS40ObserveOnceFailsPodListError(t *testing.T) { + wantErr := errors.New("pod list connection reset") + clientset := k8sfake.NewSimpleClientset() + clientset.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, wantErr + }) + env, state := s40ObserverTestEnv(t, clientset) + + observation := &s40ContinuousObservation{} + s40ObserveOnce(context.Background(), env, state, observation) + s40RequireObservationError(t, observation, "list reader pods", wantErr.Error()) +} + +func TestS40ObserveOnceFailsInternalEndpointSliceReadError(t *testing.T) { + wantErr := errors.New("internal EndpointSlice watch cache unavailable") + clientset := k8sfake.NewSimpleClientset(s40ReplacementTestPod()) + endpointReads := 0 + clientset.PrependReactor("list", "endpointslices", func(k8stesting.Action) (bool, runtime.Object, error) { + endpointReads++ + if endpointReads == 2 { + return true, nil, wantErr + } + return false, nil, nil + }) + env, state := s40ObserverTestEnv(t, clientset) + + observation := &s40ContinuousObservation{} + s40ObserveOnce(context.Background(), env, state, observation) + s40RequireObservationError(t, observation, "read reader internal EndpointSlices", wantErr.Error()) +} + +func TestS40ObserveOnceAllowsEmptyInternalEndpointSliceTransition(t *testing.T) { + env, state := s40ObserverTestEnv(t, k8sfake.NewSimpleClientset(s40ReplacementTestPod())) + observation := &s40ContinuousObservation{} + + s40ObserveOnce(context.Background(), env, state, observation) + + observation.mu.Lock() + defer observation.mu.Unlock() + if observation.err != nil { + t.Fatalf("successful empty internal EndpointSlice transition recorded error: %v", observation.err) + } + if observation.sawInternalUnreadyPod { + t.Fatal("empty internal EndpointSlice list unexpectedly recorded the replacement endpoint") + } +} + +func TestS40ObserveOnceAllowsExpectedEmptyListTransitions(t *testing.T) { + env, state := s40ObserverTestEnv(t, k8sfake.NewSimpleClientset()) + observation := &s40ContinuousObservation{} + + s40ObserveOnce(context.Background(), env, state, observation) + + observation.mu.Lock() + defer observation.mu.Unlock() + if observation.err != nil { + t.Fatalf("successful empty list transitions recorded error: %v", observation.err) + } + if !observation.sawClientEndpointEmpty { + t.Fatal("successful empty client EndpointSlice list did not record endpoint absence") + } +} + +func TestS40ObserveReadErrorIgnoresOnlyObserverCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + observation := &s40ContinuousObservation{} + s40ObserveReadError(ctx, observation, "read MFG", context.Canceled) + if observation.err != nil { + t.Fatalf("observer cancellation recorded as read failure: %v", observation.err) + } + + liveObservation := &s40ContinuousObservation{} + s40ObserveReadError(context.Background(), liveObservation, "read MFG", context.DeadlineExceeded) + s40RequireObservationError(t, liveObservation, "read MFG", context.DeadlineExceeded.Error()) +} + +func s40ObserverTestEnv(t *testing.T, clientset *k8sfake.Clientset) (*runner.Env, *s40RunState) { + t.Helper() + lag := int64(0) + mfg := &v1alpha1.MysqlFailoverGroup{ + ObjectMeta: metav1.ObjectMeta{Name: pgkube.FailoverGroupName, Namespace: pgkube.PlaygroundNamespace}, + Spec: v1alpha1.MysqlFailoverGroupSpec{ + Sites: []v1alpha1.SiteSpec{ + {Name: "iad", Role: v1alpha1.SiteRolePrimaryCandidate}, + {Name: "pdx", Role: v1alpha1.SiteRolePrimaryCandidate}, + {Name: "reader", Role: v1alpha1.SiteRoleReadOnly}, + }, + }, + Status: v1alpha1.MysqlFailoverGroupStatus{ + ActiveSite: "iad", + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}, + Sites: []v1alpha1.SiteStatus{ + {Name: "iad", State: "writable"}, + {Name: "pdx", State: "read-only", Replicating: true}, + { + Name: "reader", + State: "read-only", + Replicating: true, + SecondsBehindSource: &lag, + SourceHost: playgroundInternalSiteHost(pgkube.FailoverGroupName, "iad", pgkube.PlaygroundNamespace), + SourceConvergenceState: v1alpha1.SourceConvergenceConverged, + }, + }, + }, + } + env := testScenarioEnv(t, mfg) + env.Kube.Kubernetes = clientset + state := &s40RunState{ + reader: "reader", + active: "iad", + donorHost: playgroundInternalSiteHost(pgkube.FailoverGroupName, "iad", pgkube.PlaygroundNamespace), + originalPod: "original-pod", + } + return env, state +} + +func s40ReplacementTestPod() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reader-replacement", + Namespace: pgkube.PlaygroundNamespace, + UID: types.UID("replacement-pod"), + Labels: map[string]string{ + "app.kubernetes.io/name": "mysql", + "app.kubernetes.io/instance": pgkube.FailoverGroupName, + "shipstream.io/site": "reader", + }, + }, + } +} + +func s40RequireObservationError(t *testing.T, observation *s40ContinuousObservation, fragments ...string) { + t.Helper() + err := observation.result() + if err == nil { + t.Fatal("observation.result() = nil, want read failure") + } + for _, fragment := range fragments { + if !strings.Contains(err.Error(), fragment) { + t.Errorf("observation error %q does not contain %q", err, fragment) + } + } +} diff --git a/internal/playground/scenarios/s40_reader_data_loss.go b/internal/playground/scenarios/s40_reader_data_loss.go new file mode 100644 index 0000000..03f16f2 --- /dev/null +++ b/internal/playground/scenarios/s40_reader_data_loss.go @@ -0,0 +1,541 @@ +package scenarios + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + pglogs "github.com/shipstream/bloodraven/internal/playground/logs" + pgmysql "github.com/shipstream/bloodraven/internal/playground/mysql" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +const s40MarkerTable = "bloodraven_reader_e2e.marker" + +type s40RunState struct { + reader string + active string + donorHost string + marker string + originalPVC string + replacement string + originalPod string + replacementPod string +} + +type s40ContinuousObservation struct { + mu sync.Mutex + err error + sawClientEndpointEmpty bool + sawInternalUnreadyPod bool +} + +func init() { + runner.Register(scenario40ReaderDataLoss()) +} + +func scenario40ReaderDataLoss() runner.Scenario { + state := &s40RunState{} + return runner.Scenario{ + ID: "40-reader-data-loss-reclone", + Title: "Read-only reader data loss auto-clones without degrading the group", + Hypothesis: "Replacing the dedicated reader PVC removes its client endpoint without changing group Ready or activeSite; " + + "the operator auto-clones from the confirmed active primary and restores direct, lag-bounded replication.", + Risk: "high", + DocLink: "playground/chaos-scenarios.md#40-reader-data-loss-and-auto-clone", + Timeout: 10 * time.Minute, + ResetBeforeRunAll: true, + Precheck: s40Precheck(state), + Steps: []runner.Step{ + s40SeedAndReplicateMarker(state), + s40ReplaceReaderDataAndObserve(state), + }, + } +} + +func s40Precheck(state *s40RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + *state = s40RunState{} + if err := AssertHealthyBaseline(ctx, env); err != nil { + return err + } + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + for i := range mfg.Spec.Sites { + if mfg.Spec.Sites[i].IsReadOnlyReader() { + if state.reader != "" { + return fmt.Errorf("precheck requires exactly one read-only role site, found at least %q and %q", state.reader, mfg.Spec.Sites[i].Name) + } + state.reader = mfg.Spec.Sites[i].Name + } + } + if state.reader == "" { + return fmt.Errorf("precheck requires exactly one read-only role site") + } + state.active = mfg.Status.ActiveSite + if site := mfg.Spec.SiteByName(state.active); site == nil || !site.IsPromotable() { + return fmt.Errorf("active site %q is missing or non-promotable", state.active) + } + state.donorHost = playgroundInternalSiteHost(env.FG, state.active, env.Namespace) + + readerStatus := s40StatusSite(mfg, state.reader) + if readerStatus == nil { + return fmt.Errorf("reader %q missing from status", state.reader) + } + if err := s40AssertReaderStatus(mfg, readerStatus, state.donorHost); err != nil { + return fmt.Errorf("reader status precheck: %w", err) + } + pod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, state.reader) + if err != nil { + return err + } + node, err := env.Kube.GetNode(ctx, pod.Spec.NodeName) + if err != nil { + return fmt.Errorf("get reader node %s: %w", pod.Spec.NodeName, err) + } + if got := node.Labels["topology.kubernetes.io/zone"]; got != "zone-reader" { + return fmt.Errorf("reader pod %s scheduled in zone %q, want zone-reader", pod.Name, got) + } + if got := node.Labels["shipstream.io/site.playground"]; got != state.reader { + return fmt.Errorf("reader pod node %s site label=%q, want %q", node.Name, got, state.reader) + } + for _, site := range mfg.Spec.Sites { + if site.Name == state.reader { + continue + } + peerPod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, site.Name) + if err == nil && peerPod.Spec.NodeName == pod.Spec.NodeName { + return fmt.Errorf("reader pod shares node %s with site %s; dedicated reader worker required", pod.Spec.NodeName, site.Name) + } + } + if err := s40AssertServiceTopology(ctx, env, state.reader, pod.Name, true); err != nil { + return fmt.Errorf("reader service precheck: %w", err) + } + return nil + } +} + +func s40SeedAndReplicateMarker(state *s40RunState) runner.Step { + return runner.Step{ + Phase: runner.PhasePrecheck, + Name: "write a marker on the active primary and confirm it reaches the reader", + Do: func(ctx context.Context, env *runner.Env) error { + state.marker = fmt.Sprintf("reader-loss-%d", time.Now().UnixNano()) + active, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, state.active, env.Creds) + if err != nil { + return fmt.Errorf("open active MySQL %s: %w", state.active, err) + } + defer active.Close() + if _, err := active.Exec(ctx, "CREATE DATABASE IF NOT EXISTS bloodraven_reader_e2e; CREATE TABLE IF NOT EXISTS "+s40MarkerTable+" (marker VARCHAR(128) PRIMARY KEY); INSERT INTO "+s40MarkerTable+" (marker) VALUES (?)", state.marker); err != nil { + return fmt.Errorf("write marker %q: %w", state.marker, err) + } + return s40WaitForMarker(ctx, env, state.reader, state.marker, 60*time.Second) + }, + } +} + +func s40ReplaceReaderDataAndObserve(state *s40RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseInject, + Name: "replace reader PVC; continuously assert Ready and endpoint safety through auto-clone", + Do: func(ctx context.Context, env *runner.Env) error { + operatorLogs, err := env.Logs("operator") + if err != nil { + return fmt.Errorf("open operator log tailer: %w", err) + } + pvcName := pgkube.MysqlPVCName(env.FG, state.reader) + state.originalPVC, err = env.Kube.PVCUID(ctx, env.Namespace, pvcName) + if err != nil || state.originalPVC == "" { + return fmt.Errorf("read original reader PVC UID: uid=%q err=%v", state.originalPVC, err) + } + oldPod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, state.reader) + if err != nil { + return err + } + state.originalPod = string(oldPod.UID) + + observeCtx, stopObserve := context.WithCancel(ctx) + observation := &s40ContinuousObservation{} + observeDone := make(chan struct{}) + go func() { + defer close(observeDone) + s40ObserveContinuously(observeCtx, env, state, observation) + }() + stopAndCheck := func() error { + stopObserve() + <-observeDone + return observation.result() + } + + if err := env.Chaos.WipeSiteData(ctx, state.reader, env.FG); err != nil { + _ = stopAndCheck() + return fmt.Errorf("wipe reader data: %w", err) + } + state.replacement, err = env.Kube.PVCUID(ctx, env.Namespace, pvcName) + if err != nil || state.replacement == "" || state.replacement == state.originalPVC { + _ = stopAndCheck() + return fmt.Errorf("reader PVC was not replaced: original=%q replacement=%q err=%v", state.originalPVC, state.replacement, err) + } + if err := env.Chaos.Revert(ctx); err != nil { + _ = stopAndCheck() + return fmt.Errorf("restore reader deployment scale: %w", err) + } + + logCtx, cancelLog := context.WithTimeout(ctx, 3*time.Minute) + _, err = env.Wait.UntilLog(logCtx, operatorLogs, env.StartTime, + "reader auto-clone starts from the captured active primary internal Service", + pglogs.Structured("starting bootstrap", map[string]string{ + "source": "auto-clone", + "donor": state.active, + "recipient": state.reader, + "donorHost": state.donorHost, + }), + ) + cancelLog() + if err != nil { + _ = stopAndCheck() + return err + } + + waitCtx, cancelWait := context.WithTimeout(ctx, 6*time.Minute) + mfg, err := env.Wait.UntilCR(waitCtx, env.Namespace, "reader returns with direct healthy replication", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + if mfg.Status.ActiveSite != state.active { + return false, fmt.Sprintf("activeSite=%q", mfg.Status.ActiveSite), fmt.Errorf("active site changed during reader loss: %q -> %q", state.active, mfg.Status.ActiveSite) + } + status := s40StatusSite(mfg, state.reader) + if status == nil { + return false, "reader status missing", nil + } + err := s40AssertReaderStatus(mfg, status, state.donorHost) + return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s lag=%v", status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SecondsBehindSource), nil + }) + cancelWait() + if err != nil { + _ = stopAndCheck() + return err + } + if mfg.Status.ActiveSite != state.active { + _ = stopAndCheck() + return fmt.Errorf("active site changed during reader recovery: %q -> %q", state.active, mfg.Status.ActiveSite) + } + + if err := s40WaitForLiveReader(ctx, env, state, 90*time.Second); err != nil { + _ = stopAndCheck() + return err + } + newPod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, state.reader) + if err != nil { + _ = stopAndCheck() + return err + } + state.replacementPod = string(newPod.UID) + if state.replacementPod == state.originalPod { + _ = stopAndCheck() + return fmt.Errorf("reader pod UID did not change after PVC replacement: %s", state.originalPod) + } + if err := s40WaitForServiceTopology(ctx, env, state.reader, newPod.Name, 60*time.Second); err != nil { + _ = stopAndCheck() + return err + } + if err := stopAndCheck(); err != nil { + return err + } + env.Capture.Note(fmt.Sprintf("reader recovered: active=%s source=%s pvc=%s->%s pod=%s->%s", state.active, state.donorHost, state.originalPVC, state.replacement, state.originalPod, state.replacementPod)) + return nil + }, + } +} + +func s40ObserveContinuously(ctx context.Context, env *runner.Env, state *s40RunState, observation *s40ContinuousObservation) { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + if ctx.Err() != nil { + return + } + s40ObserveOnce(ctx, env, state, observation) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func s40ObserveOnce(ctx context.Context, env *runner.Env, state *s40RunState, observation *s40ContinuousObservation) { + mfg, mfgErr := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + s40ObserveReadError(ctx, observation, "read MFG", mfgErr) + readerServingHealthy := false + if mfgErr == nil { + if pgkube.ReadyCondition(mfg) != "True" { + observation.fail(fmt.Errorf("group Ready changed to %q while reader was unavailable/recovering", pgkube.ReadyCondition(mfg))) + } + if mfg.Status.ActiveSite != state.active { + observation.fail(fmt.Errorf("active site changed while reader was unavailable/recovering: %q -> %q", state.active, mfg.Status.ActiveSite)) + } + if status := s40StatusSite(mfg, state.reader); status != nil { + readerServingHealthy = s40AssertReaderStatus(mfg, status, state.donorHost) == nil + } + } + + clientState, clientErr := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.reader)) + s40ObserveReadError(ctx, observation, "read reader client EndpointSlices", clientErr) + if clientErr == nil { + serving := clientState.ServingPodNames("mysql") + if len(serving) == 0 { + // Empty EndpointSlice results are expected while scale-to-zero or + // endpoint turnover is in progress. The successful list, rather than + // an ignored NotFound/read error, is the explicit absence signal. + observation.markClientEmpty() + } else if mfgErr == nil && !readerServingHealthy { + observation.fail(fmt.Errorf("reader client Service published serving pods %v while reader status was unhealthy", serving)) + } + } + + pods, podErr := env.Kube.ListMysqlPods(ctx, env.Namespace, env.FG, state.reader) + s40ObserveReadError(ctx, observation, "list reader pods", podErr) + if podErr != nil { + return + } + // A successful empty pod list is the expected scale-to-zero transition. + for i := range pods.Items { + pod := &pods.Items[i] + if string(pod.UID) == state.originalPod || s40PodReady(pod) { + continue + } + if clientErr == nil && len(clientState.ServingPodNames("mysql")) != 0 { + observation.fail(fmt.Errorf("reader client Service published a serving endpoint while replacement pod %s was not ready", pod.Name)) + } + internal, internalErr := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.reader)+"-internal") + s40ObserveReadError(ctx, observation, "read reader internal EndpointSlices", internalErr) + if internalErr != nil { + continue + } + for _, endpoint := range internal.Endpoints { + if endpoint.PodName == pod.Name { + observation.markInternalUnready() + } + } + } +} + +func s40ObserveReadError(ctx context.Context, observation *s40ContinuousObservation, operation string, err error) { + if err == nil { + return + } + // Shutdown may cancel an in-flight API call after the scenario has + // explicitly stopped the observer. A context-shaped API error while the + // observer context is still live remains a real observation failure. + if ctx.Err() != nil && errors.Is(err, ctx.Err()) { + return + } + observation.fail(fmt.Errorf("continuous observer %s: %w", operation, err)) +} + +func (o *s40ContinuousObservation) fail(err error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.err == nil { + o.err = err + } +} + +func (o *s40ContinuousObservation) markClientEmpty() { + o.mu.Lock() + o.sawClientEndpointEmpty = true + o.mu.Unlock() +} + +func (o *s40ContinuousObservation) markInternalUnready() { + o.mu.Lock() + o.sawInternalUnreadyPod = true + o.mu.Unlock() +} + +func (o *s40ContinuousObservation) result() error { + o.mu.Lock() + defer o.mu.Unlock() + if o.err != nil { + return o.err + } + if !o.sawClientEndpointEmpty { + return fmt.Errorf("reader client Service was never observed without a serving endpoint during recovery") + } + if !o.sawInternalUnreadyPod { + return fmt.Errorf("reader internal Service never published the replacement pod while it was unready") + } + return nil +} + +func s40AssertReaderStatus(mfg *v1alpha1.MysqlFailoverGroup, status *v1alpha1.SiteStatus, expectedHost string) error { + if status.State != "read-only" { + return fmt.Errorf("state=%q, want read-only", status.State) + } + if !status.Replicating { + return fmt.Errorf("replicating=false") + } + if status.SourceConvergenceState != v1alpha1.SourceConvergenceConverged { + return fmt.Errorf("sourceConvergenceState=%q, want Converged", status.SourceConvergenceState) + } + if canonicalMySQLHost(status.SourceHost) != canonicalMySQLHost(expectedHost) { + return fmt.Errorf("sourceHost=%q, want %q", status.SourceHost, expectedHost) + } + if status.SecondsBehindSource == nil { + return fmt.Errorf("secondsBehindSource is unknown") + } + maxLag := mfg.Spec.EffectiveReadOnlyMaxLagSeconds() + if *status.SecondsBehindSource > maxLag { + return fmt.Errorf("secondsBehindSource=%d exceeds reader threshold %d", *status.SecondsBehindSource, maxLag) + } + return nil +} + +func s40WaitForLiveReader(ctx context.Context, env *runner.Env, state *s40RunState, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last string + for time.Now().Before(deadline) { + client, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, state.reader, env.Creds) + if err == nil { + rs, statusErr := client.ShowReplicaStatus(ctx) + markerCount, markerErr := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+s40MarkerTable+" WHERE marker=?", state.marker) + _ = client.Close() + last = fmt.Sprintf("configured=%v io=%v sql=%v source=%q lag=%v marker=%d statusErr=%v markerErr=%v", rs.Configured, rs.IORunning, rs.SQLRunning, rs.SourceHost, rs.SecondsBehindSrc, markerCount, statusErr, markerErr) + if statusErr == nil && markerErr == nil && rs.Configured && rs.IORunning && rs.SQLRunning && canonicalMySQLHost(rs.SourceHost) == canonicalMySQLHost(state.donorHost) && rs.SecondsBehindSrc != nil && markerCount == 1 { + return nil + } + } else { + last = err.Error() + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("reader live replication did not converge within %s (last: %s)", timeout, last) +} + +func s40WaitForMarker(ctx context.Context, env *runner.Env, site, marker string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + client, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, site, env.Creds) + if err == nil { + count, queryErr := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+s40MarkerTable+" WHERE marker=?", marker) + _ = client.Close() + if queryErr == nil && count == 1 { + return nil + } + last = queryErr + } else { + last = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("marker %q did not replicate to %s within %s: %v", marker, site, timeout, last) +} + +func s40AssertServiceTopology(ctx context.Context, env *runner.Env, reader, expectedPod string, requireClientEndpoint bool) error { + clientName := pgkube.MysqlDeploymentName(env.FG, reader) + clientService, err := env.Kube.Kubernetes.CoreV1().Services(env.Namespace).Get(ctx, clientName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get client Service: %w", err) + } + if len(clientService.Spec.Ports) != 1 || clientService.Spec.Ports[0].Name != "mysql" || clientService.Spec.Ports[0].Port != 3306 { + return fmt.Errorf("client Service ports=%v, want only mysql:3306", clientService.Spec.Ports) + } + clientEndpoints, err := env.Kube.ServiceEndpointState(ctx, env.Namespace, clientName) + if err != nil { + return err + } + if _, exposed := clientEndpoints.Ports["sidecar"]; exposed { + return fmt.Errorf("client Service EndpointSlice exposes sidecar port") + } + if requireClientEndpoint { + ready := clientEndpoints.ReadyPodNames("mysql") + if len(ready) != 1 || ready[0] != expectedPod { + return fmt.Errorf("client Service ready pods=%v, want exactly %s", ready, expectedPod) + } + } + + internalName := clientName + "-internal" + internalService, err := env.Kube.Kubernetes.CoreV1().Services(env.Namespace).Get(ctx, internalName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get internal Service: %w", err) + } + if internalService.Spec.Type != corev1.ServiceTypeClusterIP || !internalService.Spec.PublishNotReadyAddresses { + return fmt.Errorf("internal Service type=%s publishNotReadyAddresses=%v, want ClusterIP/true", internalService.Spec.Type, internalService.Spec.PublishNotReadyAddresses) + } + internalPorts := map[string]int32{} + for _, port := range internalService.Spec.Ports { + internalPorts[port.Name] = port.Port + } + if internalPorts["mysql"] != 3306 || internalPorts["sidecar"] != 8080 || len(internalPorts) != 2 { + return fmt.Errorf("internal Service ports=%v, want mysql:3306 and sidecar:8080", internalPorts) + } + if _, gated := internalService.Spec.Selector["shipstream.io/healthy"]; gated { + return fmt.Errorf("internal Service must not select shipstream.io/healthy") + } + return nil +} + +func s40WaitForServiceTopology(ctx context.Context, env *runner.Env, reader, expectedPod string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + if err := s40AssertServiceTopology(ctx, env, reader, expectedPod, true); err == nil { + return nil + } else { + last = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("reader Service topology did not converge within %s: %w", timeout, last) +} + +func s40StatusSite(mfg *v1alpha1.MysqlFailoverGroup, name string) *v1alpha1.SiteStatus { + for i := range mfg.Status.Sites { + if mfg.Status.Sites[i].Name == name { + return &mfg.Status.Sites[i] + } + } + return nil +} + +func playgroundInternalSiteHost(group, site, namespace string) string { + return fmt.Sprintf("mysql-%s-%s-internal.%s.svc.cluster.local", group, site, namespace) +} + +func canonicalMySQLHost(host string) string { + host = strings.ToLower(strings.TrimSpace(host)) + host = strings.TrimSuffix(host, ":3306") + return strings.TrimSuffix(host, ".") +} + +func s40PodReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/internal/state/matrix.go b/internal/state/matrix.go index ba91296..079f6e6 100644 --- a/internal/state/matrix.go +++ b/internal/state/matrix.go @@ -14,6 +14,9 @@ const ( SiteRolePrimaryCandidate SiteRole = "primary-candidate" // SiteRoleDROnly is a site that only ever follows the active primary. SiteRoleDROnly SiteRole = "dr-only" + // SiteRoleReadOnly is a non-promotable serving reader whose health does + // not contribute to failover-group readiness. + SiteRoleReadOnly SiteRole = "read-only" ) // SiteObservation is a snapshot of one site at a single poll cycle. @@ -26,6 +29,9 @@ type SiteObservation struct { // CrossSiteAction describes the cross-site action implied by a poll // cycle of observations. type CrossSiteAction struct { + // FenceSites lists writable non-promotable sites. These are anomalies, + // never active primaries or split-brain winners. + FenceSites []string // PromotionCandidates is the ordered list of primary-candidate // sites that are eligible to be promoted this cycle. The caller // must rank by GTID freshness as the primary selector (most- @@ -64,6 +70,13 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros var writable, readOnly, unreachable []SiteObservation for _, obs := range observations { + if obs.State == StateWritable && obs.Role != SiteRolePrimaryCandidate { + action.FenceSites = append(action.FenceSites, obs.Name) + continue + } + if obs.Role == SiteRoleReadOnly { + continue + } switch obs.State { case StateWritable: writable = append(writable, obs) @@ -74,13 +87,19 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros } } - if len(observations) == 0 { - action.Reason = "Healthy" + coreCount := len(writable) + len(readOnly) + len(unreachable) + if coreCount == 0 { + if len(action.FenceSites) > 0 { + action.Alert = fmt.Sprintf("writable non-promotable site requires fencing (%s)", strings.Join(action.FenceSites, ", ")) + action.Reason = "Degraded" + } else { + action.Reason = "Healthy" + } return action } // Total loss: every site is unreachable. - if len(unreachable) == len(observations) { + if len(unreachable) == coreCount { action.Alert = "TOTAL LOSS: all sites are unreachable" action.Reason = "TotalLoss" return action @@ -128,6 +147,11 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros action.Reason = "Degraded" return action } + if len(action.FenceSites) > 0 { + action.Alert = fmt.Sprintf("writable non-promotable site requires fencing (%s)", strings.Join(action.FenceSites, ", ")) + action.Reason = "Degraded" + return action + } action.Reason = "Healthy" return action } diff --git a/internal/state/matrix_test.go b/internal/state/matrix_test.go index 88fe141..31d794a 100644 --- a/internal/state/matrix_test.go +++ b/internal/state/matrix_test.go @@ -156,6 +156,70 @@ func TestEvalCrossSite_ThreeSite(t *testing.T) { }) } +func TestEvalCrossSite_ReadOnlyReaderIsolation(t *testing.T) { + pc := SiteRolePrimaryCandidate + reader := SiteRoleReadOnly + dr := SiteRoleDROnly + + tests := []struct { + name string + obs []SiteObservation + wantReason string + wantFence []string + }{ + { + name: "reader outage does not degrade healthy core", + obs: []SiteObservation{ + obs("iad", pc, StateWritable), obs("pdx", pc, StateReadOnly), obs("reader", reader, StateUnreachable), + }, + wantReason: "Healthy", + }, + { + name: "writable reader is fenced but not split brain", + obs: []SiteObservation{ + obs("iad", pc, StateWritable), obs("pdx", pc, StateReadOnly), obs("reader", reader, StateWritable), + }, + wantReason: "Degraded", + wantFence: []string{"reader"}, + }, + { + name: "writable dr-only degrades healthy candidate topology", + obs: []SiteObservation{ + obs("iad", pc, StateWritable), obs("pdx", pc, StateReadOnly), obs("dr", dr, StateWritable), + }, + wantReason: "Degraded", + wantFence: []string{"dr"}, + }, + { + name: "sole writable reader is not a primary", + obs: []SiteObservation{ + obs("iad", pc, StateReadOnly), obs("pdx", pc, StateReadOnly), obs("reader", reader, StateWritable), + }, + wantReason: "NoPrimary", + wantFence: []string{"reader"}, + }, + { + name: "writable dr only is fenced and not primary", + obs: []SiteObservation{ + obs("iad", pc, StateReadOnly), obs("pdx", pc, StateReadOnly), obs("dr", dr, StateWritable), + }, + wantReason: "NoPrimary", + wantFence: []string{"dr"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := EvalCrossSite(tt.obs, nil) + if got.Reason != tt.wantReason || !equalStrings(got.FenceSites, tt.wantFence) { + t.Fatalf("EvalCrossSite() = %+v, want reason=%s fence=%v", got, tt.wantReason, tt.wantFence) + } + if got.SplitBrain { + t.Fatal("non-promotable writable site must not create split brain") + } + }) + } +} + func TestRankPromotionCandidates(t *testing.T) { pc := SiteRolePrimaryCandidate dr := SiteRoleDROnly diff --git a/playground/chaos-scenarios.md b/playground/chaos-scenarios.md index 25d86d4..bcc8a07 100644 --- a/playground/chaos-scenarios.md +++ b/playground/chaos-scenarios.md @@ -18,7 +18,7 @@ The runner stamps an in-progress marker (`chaos.playground.bloodraven.io/in-prog - `--force` — delete any prior marker before preflight (banner printed). - `--auto-reset` — on Precheck failure, shell out to `./playground/reset-mysql.sh && ./playground/setup.sh`, then retry the scenario once. Wipes data; 3-second confirmation pause unless `CI=1`. -`chaos-check` runs the same structural baseline scenarios use (stuck scale-to-0 deployments, bogus `lastFailoverTarget`, anti-flap cooldown still ticking, both-sites-read-only `NoPrimary` symptom, replication off on a non-active candidate) and prints `inProgress: yes/no + summary`. Each error includes the exact remediation command, so `chaos-check` is the fastest way to decide whether to re-run, `--force`, or reset. +`chaos-check` runs the same structural baseline scenarios use (stuck scale-to-0 deployments, bogus `lastFailoverTarget`, anti-flap cooldown still ticking, all-sites-read-only `NoPrimary` symptom, replication off on a non-active candidate) and prints `inProgress: yes/no + summary`. Each error includes the exact remediation command, so `chaos-check` is the fastest way to decide whether to re-run, `--force`, or reset. After each scenario, standard cleanup waits for the MFG `Ready=True` condition, stable MySQL site states, and, when `spec.dragonfly.enabled=true`, `status.dragonfly.phase=Ready` with exactly one master and healthy replicas. This keeps `run-all` from starting the next scenario while Dragonfly is still recovering from an expected REPLTAKEOVER or pod restart. @@ -30,14 +30,14 @@ Currently automated (every `runner.Register` entry in `internal/playground/scena - `02-operator-kill-restart` (§2; negative-assertion — verifies activeSite stable and no SELF-FENCING during operator restart) - `02-planned-switchover` (§S; planned-failover state machine — annotates the MFG with `bloodraven.shipstream.io/planned-failover=`, asserts `Validating→Draining→WaitingForLag→Promoting→Resuming→Succeeded`, `transactionsLost==0`, and the `bloodraven_planned_failovers_total{result="success"}` increment) - `04-data-integrity-on-failover` (§4; seeds rows, blocks on `WAIT_FOR_EXECUTED_GTID_SET`, kills primary, asserts `GTID_SUBSET(pre, post)=1` and full row count on the new primary) -- `05-operator-kill-during-failover` (§5; scales the active primary to 0, sleeps 1s, kills the operator pod, asserts the cluster reconverges to 1 writable + 1 read-only with `Ready=True` and neither sidecar emitted SELF-FENC during the operator-down gap) +- `05-operator-kill-during-failover` (§5; scales the active primary to 0, sleeps 1s, kills the operator pod, asserts the cluster reconverges to 1 writable + all followers read-only with `Ready=True` and no sidecar emitted SELF-FENC during the operator-down gap) - `05-split-brain-auto-resolve` (§SBR; requires `spec.splitBrainPolicy.sitePriorities` set — clears `super_read_only` on the read-only site to force both writable, asserts the operator fences the non-preferred site, increments `bloodraven_split_brain_auto_resolve_total{prefer_site=...}`, and logs `split-brain auto-resolve`) -- `06-self-fence-isolated-primary` (§6; scales operator AND peer to 0 — true isolation path, complements `09-`) +- `06-self-fence-isolated-primary` (§6; scales operator AND every non-self peer, including the reader, to 0 — true isolation path, complements `09-`) - `08-gtid-divergence-detection` (§8; manufactures a rogue write on the old primary, asserts `recoveryState=RecoveryBlocked` + `divergentTransactionCount>0` + `divergence detected` log; auto-reclones in cleanup) - `09-anti-flap-cooldown` (§9; force-deletes the active primary, waits for failover + the original primary to come back as a reachable read-only candidate, then scales the new primary to 0 inside `failoverCooldown` and asserts `failovers_total` increments by exactly 1 across all sites — best-effort log scan for `failover blocked by anti-flap cooldown`) - `09-network-partition-self-fence` (§3 of this doc, NetworkPolicy partition path) -- `10-full-bootstrap-after-data-wipe` (§10; scales replica to 0, scrubs the per-FG readonly taint from every node, deletes the replica's data PVC, and waits for the operator's `Bootstrapping` condition to cycle through `Cloning` → `WaitingForRestart` → `SetupReplication` → `Done` with replica IO+SQL threads ON at the end) -- `11-total-loss-recovery` (§11; scales both sites to 0, asserts `TOTAL LOSS: all sites are unreachable` log + reconvergence) +- `10-full-bootstrap-after-data-wipe` (§10; scales a candidate replica to 0, scrubs the per-FG readonly taint from every node, deletes that replica's data PVC, and waits for the operator's `Bootstrapping` condition to cycle through `Cloning` → `WaitingForRestart` → `SetupReplication` → `Done` with replica IO+SQL threads ON at the end) +- `11-total-loss-recovery` (§11; scales every site to 0, asserts `TOTAL LOSS: all sites are unreachable` log + reconvergence) - `12-old-primary-recovery-no-divergence` (§7 of this doc; recovery without divergence) - `12-rolling-update-healthy-state` (§12; patches `spec.sites[*].resources.requests.memory` and asserts `status.updatePhase` engages then returns to empty, both deployments end at the new memory request, no `TOTAL LOSS` log fires during the roll) - `13-operator-kill-during-bootstrap` (§13; wipes the read-only site's PVC, scales it back up, then kills the operator pod once `Bootstrapping=True` and asserts the replacement operator drives the clone to `Status=False Reason=Done` with replica IO+SQL threads ON — runs `ResetBeforeRunAll` so `lastFailoverTarget` is empty when the precondition gate runs) @@ -68,6 +68,7 @@ Currently automated (every `runner.Register` entry in `internal/playground/scena - `37-pitr-archive-handoff-across-failover` (§37; marker A archived on the old active, emergency failover, marker B archived on the new active; a timestamp verification replays baseline+A+B and excludes C across both per-site manifests — `ResetBeforeRunAll`) - `38-dnsendpoint-write-denial-during-failover` (§38; denies the operator write verbs on `dnsendpoints`, forces promotion; the DNSEndpoint target stays stale during denial then heals to the promoted LBIP within 90s after RBAC restore with no re-failover) - `39-dragonfly-master-partition` (§39; deny-all NetworkPolicy on only the Dragonfly master pod promotes the surviving replica; MySQL `status.activeSite` and failover counters are unchanged) +- `40-reader-data-loss-reclone` (§40; release-profile reader PVC replacement, continuous `Ready=True`, auto-clone donor/internal-host log assertion, direct-source/thread/lag/marker recovery, client EndpointSlice shedding, and internal endpoint publication) Scenarios `32`–`39` are **full-profile-only** by allowlist omission — none are in the `smoke` or `release` subsets — until they accumulate broader repeated live-pass history with no destructive leakage. As of 2026-07-11, all eight have passed on the k3d playground; scenarios `34`, `35`, `36`, and `38` also passed post-review reruns with the final code. Scenario `36` needed three diagnosed-and-fixed failed attempts first (see §36's "Actual live result"). All eight remain full-profile-only while this new, high-risk coverage matures. Scenarios `36` and `37` additionally run `ResetBeforeRunAll` because they exercise backup/PITR/restore-in-place state that can leak into later scenarios if cleanup is interrupted. @@ -79,9 +80,9 @@ The runner refuses to mutate any kubectl context that does not match the same al ## Prerequisites -1. Create k3d cluster: `k3d cluster create bloodraven --agents 2 --k3s-arg '--tls-san=@server:0'` +1. Create k3d cluster: `k3d cluster create bloodraven --agents 3 --k3s-arg '--tls-san=@server:0'` 2. Run playground setup: `./playground/setup.sh` -3. Verify healthy state: `kubectl -n bloodraven-playground get mysqlfailovergroups -o wide` shows one writable, one read-only +3. Verify healthy state: `kubectl -n bloodraven-playground get mysqlfailovergroups -o wide` shows one writable and two read-only followers Key playground config: `pollInterval=2s`, `failureThreshold=3` (~6s detection), `failoverCooldown=30s`, sidecar `leaseTimeout=20s`, `peerCheckInterval=5s`. @@ -123,7 +124,7 @@ Key playground config: `pollInterval=2s`, `failureThreshold=3` (~6s detection), **Injection**: `./playground/chaos.sh kill-operator` -**Verify**: activeSite unchanged, both MySQL sites unchanged, zero "SELF-FENC" in sidecar logs, operator logs show clean startup. +**Verify**: activeSite unchanged, every MySQL site unchanged, zero "SELF-FENC" in sidecar logs, operator logs show clean startup. --- @@ -177,7 +178,7 @@ kubectl -n bloodraven-playground exec deploy/mysql-playground- -c mysql ### 6. Self-Fencing Validation **Category**: Split-brain prevention | **Risk**: Medium -**Hypothesis**: Fully isolate primary (scale operator=0, scale replica=0). Sidecar self-fences at ~T+20s. +**Hypothesis**: Fully isolate primary (scale operator=0, scale every non-self peer including the reader=0). Sidecar self-fences at ~T+20s. **Injection**: ```bash @@ -266,14 +267,14 @@ The anti-flap code (topology.go) is unit-tested but requires a scenario where on ### 11. Simultaneous Both-Site Kill **Category**: Total loss / graceful degradation | **Risk**: High -**Hypothesis**: Both MySQL deployments scaled to 0 -> TOTAL LOSS alert fires, no panic -> both come back -> operator re-establishes topology without manual intervention. +**Hypothesis**: Every MySQL deployment scaled to 0 -> TOTAL LOSS alert fires, no panic -> all come back -> operator re-establishes topology without manual intervention. **Injection**: ```bash NS=bloodraven-playground -kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx --replicas=0 +kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx mysql-playground-reader --replicas=0 # Wait 15s for TOTAL LOSS alert -kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx --replicas=1 +kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx mysql-playground-reader --replicas=1 ``` **Verify**: Operator logs "TOTAL LOSS: both sites are unreachable" during outage. After recovery, one site writable, one read-only, replication running. No operator crash or permanent degraded state. @@ -706,7 +707,7 @@ Fail-back behavior is intentionally GTID-freshest/current-state-driven. When an **Injection**: `make chaos-run SCENARIO=22-replication-status-after-recovery`. The runner scales the active primary to 0, waits for failover, scales it back up, waits for re-convergence, then reads the CR. -**Verify**: Within 30s of `writable=1 read-only=1`, `mfg.Status.Sites[i].Replicating == true` for the read-only site. The scenario also probes the sidecar `/status` endpoint as a cross-check; if the sidecar reports replication running but the CR field is false, that is exactly the bug WISHLIST #36 describes. +**Verify**: Within 30s of `writable=1 read-only=N-1`, `mfg.Status.Sites[i].Replicating == true` for the recovered candidate follower. The scenario also probes the sidecar `/status` endpoint as a cross-check; if the sidecar reports replication running but the CR field is false, that is exactly the bug WISHLIST #36 describes. **Cleanup**: Standard runner cleanup. No manual steps required. @@ -887,7 +888,7 @@ Three of these (§32, §36, §38) exposed a real operator defect and shipped a m ### 34. Operator Kill During Ordered-Update WaitReplica **Category**: Ordered update / operator resilience | **Risk**: High | **Profile**: full-only -**Hypothesis**: Killing the operator while an ordered update is held at `WaitReplica` does not double-roll both MySQL pods or cause `TOTAL LOSS`. The replacement operator, whose in-memory update phase was lost, re-derives remaining drift from the Deployment spec-hash mismatch and completes the roll. +**Hypothesis**: Killing the operator while an ordered update is held at `WaitReplica` does not double-roll all MySQL pods or cause `TOTAL LOSS`. The replacement operator, whose in-memory update phase was lost, re-derives remaining drift from the Deployment spec-hash mismatch and completes the roll. **Injection**: Patch `spec.sites[*].resources.requests.memory`; wait for `status.updatePhase` to engage (preferably `WaitReplica`); apply an **ingress-only** deny NetworkPolicy to the standby MySQL pod so the operator's health check of the freshly-rolled standby fails (egress/replication stays open, satisfying the updater's replicating-standby precondition once lifted); force-delete the operator pod; after the replacement is Available, remove the hold. @@ -1026,6 +1027,7 @@ These two scenarios cover state-machine paths that don't fit the §1-§30 emerge - `status.plannedFailover.phase` reaches `Succeeded` with `target=` (stale plannedFailover blocks left over from prior runs are ignored via the `StartTime` check, so this assertion is robust under rerun). - `status.plannedFailover.transactionsLost` is populated and equals 0. +- Direct `SHOW REPLICA STATUS` on every non-active follower reports both threads running and `Source_Host=mysql-playground--internal.bloodraven-playground.svc.cluster.local`; this explicitly covers the demoted old primary and the reader. - `bloodraven_planned_failovers_total{target_site=,result="success"}` increments. **Note**: Single-scenario `run` will fail precheck if the cluster already has an in-flight `plannedFailover` in a non-terminal phase. The runner's standard cleanup waits for `Ready=True` before the next scenario starts, so `run-all` does not need to reset between this and the next entry. @@ -1057,6 +1059,27 @@ kubectl -n bloodraven-playground patch mysqlfailovergroup playground --type json --- +### 40. Reader Data Loss and Auto-Clone {#40-reader-data-loss-and-auto-clone} +**Category**: Reader recovery and endpoint safety | **Risk**: High + +**Hypothesis**: Losing only the dedicated `read-only` reader's local data does not affect failover-group readiness or the writable primary. The operator removes the unhealthy reader from its client Service, recreates it from the confirmed active primary, converges it to a direct source, and republishes it only after threads and lag are healthy. + +**Injection**: `make chaos-run SCENARIO=40-reader-data-loss-reclone`. The scenario verifies that the reader is on the `zone-reader` worker, scales its Deployment to zero, deletes and observes replacement of its PVC UID, then scales it back to one. This deterministic local-PV replacement is the playground interpretation of reader node/storage loss; it avoids relying on literal node deletion and cloud volume behavior. + +**Verify**: + +- Exactly one spec site has effective role `read-only`; its pod is on a dedicated `zone-reader` worker. +- A marker written to the active primary is visible on the reader before loss and after recovery. +- A continuous observer fails immediately if the group `Ready` condition is ever not `True` during scale-down, empty-datadir detection, clone, or catch-up. +- Operator logs contain `starting bootstrap` with `source=auto-clone`, `donor=`, `recipient=reader`, and `donorHost=mysql-playground--internal.bloodraven-playground.svc.cluster.local`. +- The active site never changes. Final reader status is `State=read-only`, `SourceConvergenceState=Converged`, direct `SourceHost`, known lag at or below `readOnlyMaxLagSeconds`, and live MySQL IO/SQL threads running. +- The reader client Service exposes only MySQL and has no serving EndpointSlice target while unhealthy. After convergence it has exactly the replacement reader pod. +- The reader internal Service remains `ClusterIP`, exposes MySQL and sidecar, has `publishNotReadyAddresses=true`, does not use the healthy selector, and publishes the replacement pod while it is unready. + +**Cleanup**: The PVC is intentionally replaced. The Deployment scale is restored immediately and the scenario does not finish until direct healthy replication and the client endpoint return. Standard runner forensics and cleanup remain active on every failure path. + +--- + ## Execution Plan 1. **Setup**: `k3d cluster create` + `./playground/setup.sh` diff --git a/playground/chaos.sh b/playground/chaos.sh index dfd469e..790b36d 100755 --- a/playground/chaos.sh +++ b/playground/chaos.sh @@ -2,14 +2,14 @@ # Bloodraven Playground — Chaos Monkey # # Usage: -# ./playground/chaos.sh kill-site Kill MySQL+Dragonfly pods at a site +# ./playground/chaos.sh kill-site Kill MySQL+Dragonfly pods at a site # ./playground/chaos.sh kill-operator Kill the operator pod # ./playground/chaos.sh kill-counter Kill the counter app pod -# ./playground/chaos.sh kill-dragonfly Kill the Dragonfly pod at a site (StatefulSet recreates) +# ./playground/chaos.sh kill-dragonfly Kill the Dragonfly pod at a site (StatefulSet recreates) # ./playground/chaos.sh dragonfly-status Show Dragonfly status, pod roles, active endpoints -# ./playground/chaos.sh cordon Cordon the node for a site +# ./playground/chaos.sh cordon Cordon the node for a site # ./playground/chaos.sh uncordon Uncordon all nodes -# ./playground/chaos.sh network-partition Simulate network partition (drop MySQL traffic) +# ./playground/chaos.sh network-partition Simulate network partition (drop MySQL traffic) # ./playground/chaos.sh recover Undo all chaos (uncordon, restore network) # ./playground/chaos.sh status Show current state # ./playground/chaos.sh watch Continuous watch @@ -31,36 +31,26 @@ usage() { echo "Usage: $0 [args]" echo "" echo "Commands:" - echo " kill-site Delete MySQL+Dragonfly pods at the given site" + echo " kill-site Delete MySQL+Dragonfly pods at the given site" echo " kill-operator Delete the Bloodraven operator pod" echo " kill-counter Delete the counter app pod" - echo " kill-dragonfly Delete the Dragonfly pod at the given site" + echo " kill-dragonfly Delete the Dragonfly pod at the given site" echo " dragonfly-status Show Dragonfly status, pod roles, and active endpoints" - echo " cordon Cordon the node hosting the given site" + echo " cordon Cordon the node hosting the given site" echo " uncordon Uncordon all playground nodes" - echo " network-partition Block MySQL traffic on a site's node via exec into a debug pod" + echo " network-partition Block MySQL traffic to a site's pod with NetworkPolicy" echo " recover Undo all chaos actions" echo " status Show cluster and failover group status" echo " watch Watch pods and failover group continuously" exit 1 } -site_zone() { - case "$1" in - iad) echo "zone-iad" ;; - pdx) echo "zone-pdx" ;; - *) echo "Unknown site: $1" >&2; exit 1 ;; - esac -} - site_node() { - local zone - zone=$(site_zone "$1") - kubectl get nodes -l "topology.kubernetes.io/zone=$zone" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null + kubectl get nodes -l "shipstream.io/site.playground=$1" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null } cmd_kill_site() { - local site="${1:?Usage: kill-site }" + local site="${1:?Usage: kill-site }" info "Killing MySQL pod at site '$site'..." kubectl -n "$NAMESPACE" delete pod -l "shipstream.io/site=$site" --grace-period=0 --force 2>/dev/null || \ kubectl -n "$NAMESPACE" delete pod -l "shipstream.io/site=$site" @@ -82,7 +72,7 @@ cmd_kill_counter() { } cmd_cordon() { - local site="${1:?Usage: cordon }" + local site="${1:?Usage: cordon }" local node node=$(site_node "$site") if [[ -z "$node" ]]; then @@ -103,7 +93,7 @@ cmd_uncordon() { } cmd_network_partition() { - local site="${1:?Usage: network-partition }" + local site="${1:?Usage: network-partition }" info "Simulating network partition on site '$site' (NetworkPolicy deny-all)..." # Use a Kubernetes NetworkPolicy to block all ingress and egress traffic @@ -134,7 +124,7 @@ EOF } cmd_kill_dragonfly() { - local site="${1:?Usage: kill-dragonfly }" + local site="${1:?Usage: kill-dragonfly }" info "Killing Dragonfly pod at site '$site' (StatefulSet will recreate)..." # StatefulSet pods are named -; we have replicas=1 so it's always -0. local pod="playground-dragonfly-${site}-0" diff --git a/playground/manifests/failovergroup.yaml b/playground/manifests/failovergroup.yaml index e4a398f..f8a327d 100644 --- a/playground/manifests/failovergroup.yaml +++ b/playground/manifests/failovergroup.yaml @@ -51,14 +51,36 @@ spec: limits: cpu: 500m memory: 1Gi + - name: reader + role: read-only + zone: zone-reader + mysqlConf: + max-connections: "150" + serviceTemplate: + type: NodePort + externalTrafficPolicy: Local + nodePort: 30306 + annotations: + playground.bloodraven.io/purpose: site-local-reads + storage: + storageClassName: playground-fast + size: 1Gi + resources: + requests: + cpu: 100m + memory: 384Mi + limits: + cpu: 500m + memory: 1Gi sidecar: leaseTimeout: 20s peerCheckInterval: 5s replication: maxLagSeconds: 30 + readOnlyMaxLagSeconds: 10 # Playground-only Dragonfly co-management. Bloodraven creates one - # Dragonfly StatefulSet per site and configures the non-active site - # as a replica of the active site. Disable by removing this block or + # Dragonfly StatefulSet per site and configures non-active sites as + # replicas of the active site. Disable by removing this block or # setting enabled=false; the rest of the failover group is unaffected. dragonfly: enabled: true diff --git a/playground/rebuild.sh b/playground/rebuild.sh index ff23afb..efdf38b 100755 --- a/playground/rebuild.sh +++ b/playground/rebuild.sh @@ -89,8 +89,15 @@ if want sidecar; then info "Building sidecar image..." $RUNTIME build --target sidecar -t bloodraven-sidecar:playground "$PROJECT_ROOT" IMAGES+=(bloodraven-sidecar:playground) - # Sidecar runs inside MySQL pods — need to restart those - DEPLOYMENTS+=(mysql-playground-iad mysql-playground-pdx) + # Sidecar runs inside every MySQL pod; derive the deployments from the CR. + mapfile -t mysql_sites < <(kubectl -n "$NAMESPACE" get mysqlfailovergroup playground \ + -o jsonpath='{range .spec.sites[*]}{.name}{"\n"}{end}') + if [[ ${#mysql_sites[@]} -eq 0 ]]; then + fail "MysqlFailoverGroup playground has no sites; run ./playground/setup.sh first" + fi + for site in "${mysql_sites[@]}"; do + DEPLOYMENTS+=("mysql-playground-$site") + done fi if want counter; then diff --git a/playground/setup.sh b/playground/setup.sh index 3836466..0c18858 100755 --- a/playground/setup.sh +++ b/playground/setup.sh @@ -6,7 +6,7 @@ # Usage: ./playground/setup.sh # # Prerequisites: -# - kubectl pointing at a cluster with at least 2 nodes +# - kubectl pointing at a cluster with at least 3 worker nodes # - helm # - docker or podman (docker preferred — k3d's podman support is # experimental, and the image-load path is faster on docker). @@ -16,10 +16,10 @@ # Cluster setup (do once, before running this script): # # k3d: -# k3d cluster create bloodraven --agents 2 +# k3d cluster create bloodraven --agents 3 # # minikube: -# minikube start --nodes=2 --cpus=2 --memory=2048 --driver=docker +# minikube start --nodes=4 --cpus=2 --memory=2048 --driver=docker # # kind: # cat <\033[0m $*"; } ok() { echo -e "\033[1;32m OK\033[0m $*"; } @@ -94,36 +98,29 @@ fi kubectl cluster-info >/dev/null 2>&1 || fail "No Kubernetes cluster reachable. Set up a cluster first (see script header)." -# ── 1. Verify at least 2 nodes ─────────────────────────────────────────── -NODES=($(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name)) -if [[ ${#NODES[@]} -lt 2 ]]; then - fail "Need at least 2 nodes, got ${#NODES[@]}. See script header for cluster setup." -fi -ok "Cluster reachable with ${#NODES[@]} nodes" - -# ── 2. Label nodes to simulate sites ──────────────────────────────────── -info "Labeling nodes as site zones..." -# Pick first two worker nodes (skip control-plane-only if possible) +# ── 1. Verify at least 3 worker nodes ──────────────────────────────────── +mapfile -t NODES < <(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name) WORKERS=() for n in "${NODES[@]}"; do labels=$(kubectl get node "$n" --show-labels --no-headers 2>/dev/null || true) - if [[ "$labels" != *"node-role.kubernetes.io/control-plane"* && "$labels" != *"node-role.kubernetes.io/master"* ]]; then + unschedulable=$(kubectl get node "$n" -o jsonpath='{.spec.unschedulable}' 2>/dev/null || true) + if [[ "$unschedulable" != "true" && "$labels" != *"node-role.kubernetes.io/control-plane"* && "$labels" != *"node-role.kubernetes.io/master"* ]]; then WORKERS+=("$n") fi done -# Fall back to all nodes if no pure workers -if [[ ${#WORKERS[@]} -lt 2 ]]; then - WORKERS=("${NODES[@]}") +if [[ ${#WORKERS[@]} -lt "$EXPECTED_SITE_COUNT" ]]; then + fail "Need at least $EXPECTED_SITE_COUNT worker nodes, got ${#WORKERS[@]}. See script header for cluster setup." fi +ok "Cluster reachable with ${#WORKERS[@]} worker nodes" -kubectl label node "${WORKERS[0]}" topology.kubernetes.io/zone=zone-iad --overwrite -kubectl label node "${WORKERS[0]}" shipstream.io/failover-group.playground=true --overwrite -kubectl label node "${WORKERS[0]}" shipstream.io/site.playground=iad --overwrite - -kubectl label node "${WORKERS[1]}" topology.kubernetes.io/zone=zone-pdx --overwrite -kubectl label node "${WORKERS[1]}" shipstream.io/failover-group.playground=true --overwrite -kubectl label node "${WORKERS[1]}" shipstream.io/site.playground=pdx --overwrite -ok "Nodes labeled: ${WORKERS[0]}=iad, ${WORKERS[1]}=pdx" +# ── 2. Label nodes to simulate sites ──────────────────────────────────── +info "Labeling nodes as site zones..." +for i in "${!SITE_NAMES[@]}"; do + kubectl label node "${WORKERS[$i]}" "topology.kubernetes.io/zone=${SITE_ZONES[$i]}" --overwrite + kubectl label node "${WORKERS[$i]}" shipstream.io/failover-group.playground=true --overwrite + kubectl label node "${WORKERS[$i]}" "shipstream.io/site.playground=${SITE_NAMES[$i]}" --overwrite +done +ok "Nodes labeled: ${WORKERS[0]}=iad, ${WORKERS[1]}=pdx, ${WORKERS[2]}=reader" # ── 3. Build images ────────────────────────────────────────────────────── if [[ -n "${SKIP_IMAGE_BUILD:-}" ]]; then @@ -343,13 +340,13 @@ for i in $(seq 1 180); do PVCS_BOUND=$(kubectl -n "$NAMESPACE" get pvc -l app.kubernetes.io/name=mysql \ -o jsonpath='{range .items[*]}{.status.phase}{"\n"}{end}' 2>/dev/null \ | grep -c '^Bound$' || true) - if [[ "$PVCS_BOUND" -ge 2 ]]; then - ok "Both MySQL PVCs are bound" + if [[ "$PVCS_BOUND" -ge "$EXPECTED_SITE_COUNT" ]]; then + ok "All $EXPECTED_SITE_COUNT MySQL PVCs are bound" break fi sleep 1 done -if [[ "$PVCS_BOUND" -lt 2 ]]; then +if [[ "$PVCS_BOUND" -lt "$EXPECTED_SITE_COUNT" ]]; then warn "Timed out waiting for MySQL PVCs to bind" kubectl -n "$NAMESPACE" get pvc -o wide 2>/dev/null || true kubectl -n "$NAMESPACE" describe pvc 2>/dev/null || true @@ -367,24 +364,24 @@ for i in $(seq 1 36); do READY=$(kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name=mysql \ -o jsonpath='{range .items[*]}{.status.containerStatuses[*].ready}{"\n"}{end}' 2>/dev/null \ | grep -c "true true" || true) - if [[ "$READY" -ge 2 ]]; then - ok "Both MySQL pods are ready" + if [[ "$READY" -ge "$EXPECTED_SITE_COUNT" ]]; then + ok "All $EXPECTED_SITE_COUNT MySQL pods are ready" break fi sleep 5 done -if [[ "$READY" -lt 2 ]]; then +if [[ "$READY" -lt "$EXPECTED_SITE_COUNT" ]]; then warn "Timed out waiting for MySQL pods" kubectl -n "$NAMESPACE" get pods,pvc -o wide 2>/dev/null || true kubectl -n "$NAMESPACE" describe pods -l app.kubernetes.io/name=mysql 2>/dev/null || true exit 1 fi -info "Creating replication user on both MySQL sites..." +info "Creating replication user on every MySQL site..." REPL_USER=$(kubectl -n "$NAMESPACE" get secret mysql-credentials -o jsonpath='{.data.MYSQL_REPLICATION_USER}' | base64 -d) REPL_PASS=$(kubectl -n "$NAMESPACE" get secret mysql-credentials -o jsonpath='{.data.MYSQL_REPLICATION_PASSWORD}' | base64 -d) ROOT_PASS=$(kubectl -n "$NAMESPACE" get secret mysql-credentials -o jsonpath='{.data.MYSQL_ROOT_PASSWORD}' | base64 -d) -for site in iad pdx; do +for site in "${SITE_NAMES[@]}"; do CREATED=false LAST_ERR="" for attempt in $(seq 1 12); do @@ -443,7 +440,7 @@ DF_ENABLED=$(kubectl -n "$NAMESPACE" get mysqlfailovergroup playground \ -o jsonpath='{.spec.dragonfly.enabled}' 2>/dev/null || echo "") if [[ "$DF_ENABLED" == "true" ]]; then info "Waiting for Dragonfly StatefulSets (one per site)..." - for site in iad pdx; do + for site in "${SITE_NAMES[@]}"; do if ! kubectl -n "$NAMESPACE" rollout status statefulset/playground-dragonfly-$site --timeout=120s 2>/dev/null; then warn "Dragonfly StatefulSet for $site did not become ready in 120s — check 'kubectl describe statefulset playground-dragonfly-$site'" fi @@ -496,6 +493,7 @@ fi echo " Chaos monkey:" echo " ./playground/chaos.sh kill-site iad" echo " ./playground/chaos.sh kill-site pdx" +echo " ./playground/chaos.sh kill-site reader" echo " ./playground/chaos.sh cordon iad" echo " ./playground/chaos.sh network-partition iad" echo " ./playground/chaos.sh kill-dragonfly iad" diff --git a/test/component/helpers_test.go b/test/component/helpers_test.go index b877789..767c579 100644 --- a/test/component/helpers_test.go +++ b/test/component/helpers_test.go @@ -105,6 +105,10 @@ func (m *mockMySQL) ChangeReplicationSource(_ context.Context, opts mysql.Replic defer m.mu.Unlock() m.replicationSourceSet = true m.changeReplicationOpts = opts + if m.replicaStatus == nil { + m.replicaStatus = &mysql.ReplicaStatus{} + } + m.replicaStatus.SourceHost = opts.Host return nil } @@ -112,6 +116,10 @@ func (m *mockMySQL) StartReplica(_ context.Context) error { m.mu.Lock() defer m.mu.Unlock() m.replicaStarted = true + if m.replicaStatus != nil { + m.replicaStatus.IORunning = true + m.replicaStatus.SQLRunning = true + } return nil } diff --git a/test/component/source_convergence_test.go b/test/component/source_convergence_test.go new file mode 100644 index 0000000..98dc5eb --- /dev/null +++ b/test/component/source_convergence_test.go @@ -0,0 +1,42 @@ +package component + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/shipstream/bloodraven/internal/controller" + "github.com/shipstream/bloodraven/internal/mysql" + "github.com/shipstream/bloodraven/internal/platform" + "github.com/shipstream/bloodraven/internal/state" +) + +func TestSourceConvergenceAfterRestartWithoutFailoverHistory(t *testing.T) { + gtid := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-10" + primary := &mockMySQL{readOnly: false, gtidExecuted: gtid} + candidate := &mockMySQL{readOnly: true, gtidExecuted: gtid, replicaStatus: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "old-hop"}} + reader := &mockMySQL{readOnly: true, gtidExecuted: gtid, replicaStatus: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "candidate-internal"}} + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + tm := controller.NewTopologyManager(controller.TopologyConfig{ + Name: "lion", FailureThreshold: 1, RecoveryThreshold: 1, ReadOnlyMaxLagSeconds: 30, + Sites: []controller.SiteTopologyConfig{ + {Name: "primary", Role: state.SiteRolePrimaryCandidate, Host: "primary-internal"}, + {Name: "candidate", Role: state.SiteRolePrimaryCandidate, Host: "candidate-internal"}, + {Name: "reader", Role: state.SiteRoleReadOnly, Host: "reader-internal"}, + }, + }, []mysql.Checker{primary, candidate, reader}, controller.NewFailoverController(logger), nil, nil, + controller.BootstrapConfig{ReplUser: "repl", ReplPassword: "secret"}, newMockTainter(), platform.NewHub(logger), nil, logger) + + tm.Poll(context.Background()) + for name, follower := range map[string]*mockMySQL{"candidate": candidate, "reader": reader} { + follower.mu.Lock() + host := follower.replicaStatus.SourceHost + changed := follower.replicationSourceSet + reset := follower.resetReplicaAll + follower.mu.Unlock() + if !changed || host != "primary-internal" || reset { + t.Fatalf("%s changed=%v source=%q reset=%v", name, changed, host, reset) + } + } +} diff --git a/test/envtest/reconciler_test.go b/test/envtest/reconciler_test.go index d7c3565..fbc77c0 100644 --- a/test/envtest/reconciler_test.go +++ b/test/envtest/reconciler_test.go @@ -3,6 +3,7 @@ package envtest import ( + "fmt" "testing" "time" @@ -114,6 +115,98 @@ func TestEnvtest_CRCreationAndSchemaAcceptance(t *testing.T) { } } +func TestEnvtest_ReadOnlyRoleValidation(t *testing.T) { + readerSite := func() v1alpha1.SiteSpec { + return v1alpha1.SiteSpec{ + Name: "reader", Role: v1alpha1.SiteRoleReadOnly, Zone: "reader-zone", + Storage: v1alpha1.StorageSpec{StorageClassName: "standard", Size: resource.MustParse("10Gi")}, + } + } + tests := []struct { + name string + mutate func(*v1alpha1.MysqlFailoverGroup) + wantOK bool + }{ + {name: "reader omits placement", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { fg.Spec.Sites = append(fg.Spec.Sites, readerSite()) }, wantOK: true}, + {name: "reader accepts legacy placement", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + r := readerSite() + r.LBIP = "203.0.113.3" + r.TaintNodeSelector = map[string]string{"site": "reader"} + fg.Spec.Sites = append(fg.Spec.Sites, r) + }, wantOK: true}, + {name: "candidate requires lb ip", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { fg.Spec.Sites[0].LBIP = "" }}, + {name: "dr only requires selector", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Spec.Sites[0].Role = v1alpha1.SiteRoleDROnly + fg.Spec.Sites[0].TaintNodeSelector = nil + }}, + {name: "reader does not satisfy candidate minimum", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Spec.Sites[1] = readerSite() + }}, + {name: "reader rejected from priorities", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Spec.Sites = append(fg.Spec.Sites, readerSite()) + fg.Spec.SplitBrainPolicy = &v1alpha1.SplitBrainPolicySpec{SitePriorities: []string{"reader"}} + }}, + {name: "valid inherited service exposure", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + fg.Spec.ServiceTemplate = &v1alpha1.ServiceTemplate{Type: corev1.ServiceTypeNodePort, ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyLocal} + r := readerSite() + r.ServiceTemplate = &v1alpha1.SiteServiceTemplate{NodePort: 31000} + fg.Spec.Sites = append(fg.Spec.Sites, r) + }, wantOK: true}, + {name: "node port rejected for cluster ip", mutate: func(fg *v1alpha1.MysqlFailoverGroup) { + r := readerSite() + r.ServiceTemplate = &v1alpha1.SiteServiceTemplate{NodePort: 31000} + fg.Spec.Sites = append(fg.Spec.Sites, r) + }}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns := fmt.Sprintf("reader-validation-%d", i) + ensureNamespace(t, ns) + fg := newTestFG(ns) + tt.mutate(fg) + err := k8sClient.Create(ctx, fg) + if tt.wantOK && err != nil { + t.Fatalf("Create() rejected valid object: %v", err) + } + if !tt.wantOK && err == nil { + t.Fatal("Create() accepted invalid object") + } + }) + } +} + +func TestEnvtest_ReadOnlyRoleTransitions(t *testing.T) { + ns := "reader-role-transitions" + ensureNamespace(t, ns) + fg := newTestFG(ns) + fg.Spec.Sites = append(fg.Spec.Sites, v1alpha1.SiteSpec{ + Name: "reader", Role: v1alpha1.SiteRoleReadOnly, Zone: "reader-zone", + Storage: v1alpha1.StorageSpec{StorageClassName: "standard", Size: resource.MustParse("10Gi")}, + }) + if err := k8sClient.Create(ctx, fg); err != nil { + t.Fatal(err) + } + fg.Spec.Sites[2].Role = v1alpha1.SiteRoleDROnly + if err := k8sClient.Update(ctx, fg); err == nil { + t.Fatal("reader to dr-only transition without placement was accepted") + } + if err := k8sClient.Get(ctx, types.NamespacedName{Name: fg.Name, Namespace: ns}, fg); err != nil { + t.Fatal(err) + } + fg.Spec.Sites[2].Role = v1alpha1.SiteRoleDROnly + fg.Spec.Sites[2].LBIP = "203.0.113.3" + fg.Spec.Sites[2].TaintNodeSelector = map[string]string{"site": "reader"} + if err := k8sClient.Update(ctx, fg); err != nil { + t.Fatalf("reader to dr-only transition with placement rejected: %v", err) + } + fg.Spec.Sites[2].Role = v1alpha1.SiteRoleReadOnly + fg.Spec.Sites[2].LBIP = "" + fg.Spec.Sites[2].TaintNodeSelector = nil + if err := k8sClient.Update(ctx, fg); err != nil { + t.Fatalf("dr-only to reader transition rejected: %v", err) + } +} + func TestEnvtest_StatusSubresourceWrites(t *testing.T) { ns := "envtest-status-write" ensureNamespace(t, ns) @@ -273,17 +366,15 @@ func TestEnvtest_ReconcilerCreatesResources(t *testing.T) { t.Fatalf("reconcile failed: %v", err) } - // Verify ConfigMap was created with owner reference - var cm corev1.ConfigMap - if err := k8sClient.Get(ctx, types.NamespacedName{ - Name: "mysql-lion-config", Namespace: ns, - }, &cm); err != nil { - t.Fatalf("ConfigMap not created: %v", err) - } - if len(cm.OwnerReferences) == 0 { - t.Error("ConfigMap should have owner reference to MysqlFailoverGroup") - } else if cm.OwnerReferences[0].Name != "lion" { - t.Errorf("ConfigMap owner ref: got %q, want 'lion'", cm.OwnerReferences[0].Name) + // Verify per-site ConfigMaps were created with owner references. + for _, dc := range []string{"dc1", "dc2"} { + var cm corev1.ConfigMap + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "mysql-lion-" + dc + "-config", Namespace: ns}, &cm); err != nil { + t.Fatalf("ConfigMap for %s not created: %v", dc, err) + } + if len(cm.OwnerReferences) == 0 || cm.OwnerReferences[0].Name != "lion" { + t.Errorf("ConfigMap %s owner refs = %v", dc, cm.OwnerReferences) + } } // Verify Deployments @@ -308,7 +399,7 @@ func TestEnvtest_ReconcilerCreatesResources(t *testing.T) { } // Verify Services - for _, svcName := range []string{"mysql-lion-dc1", "mysql-lion-dc2", "mysql-lion-primary", "mysql-lion-replicas"} { + for _, svcName := range []string{"mysql-lion-dc1", "mysql-lion-dc2", "mysql-lion-dc1-internal", "mysql-lion-dc2-internal", "mysql-lion-primary", "mysql-lion-replicas"} { var svc corev1.Service if err := k8sClient.Get(ctx, types.NamespacedName{ Name: svcName, Namespace: ns, @@ -369,7 +460,7 @@ func TestEnvtest_ReconcilerIdempotent(t *testing.T) { } } -func TestEnvtest_ReconcilerHandlesSpecChange(t *testing.T) { +func TestEnvtest_ReconcilerDefersExistingDeploymentSpecChange(t *testing.T) { ns := "envtest-spec-change" ensureNamespace(t, ns) ensureSecret(t, ns) @@ -391,7 +482,13 @@ func TestEnvtest_ReconcilerHandlesSpecChange(t *testing.T) { t.Fatalf("initial reconcile failed: %v", err) } - // Change the image + var before appsv1.Deployment + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "mysql-lion-dc1", Namespace: ns}, &before); err != nil { + t.Fatalf("deployment not found after initial reconcile: %v", err) + } + initialImage := before.Spec.Template.Spec.Containers[0].Image + + // Change the image. var fetched v1alpha1.MysqlFailoverGroup if err := k8sClient.Get(ctx, nn, &fetched); err != nil { t.Fatalf("failed to get fg: %v", err) @@ -401,20 +498,21 @@ func TestEnvtest_ReconcilerHandlesSpecChange(t *testing.T) { t.Fatalf("failed to update fg: %v", err) } - // Re-reconcile should update the deployment + // Bulk reconciliation must defer the existing Deployment to the ordered + // updater rather than patching every site concurrently. if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: nn}); err != nil { t.Fatalf("re-reconcile after spec change failed: %v", err) } - // Verify the deployment was updated + // Verify the existing Deployment was left untouched. var deploy appsv1.Deployment if err := k8sClient.Get(ctx, types.NamespacedName{ Name: "mysql-lion-dc1", Namespace: ns, }, &deploy); err != nil { t.Fatalf("deployment not found: %v", err) } - if deploy.Spec.Template.Spec.Containers[0].Image != "mysql:8.4" { - t.Errorf("expected image mysql:8.4, got %s", deploy.Spec.Template.Spec.Containers[0].Image) + if got := deploy.Spec.Template.Spec.Containers[0].Image; got != initialImage { + t.Errorf("existing deployment was patched outside ordered update: image %s -> %s", initialImage, got) } } From 639f87eab9d345f4c108ed49d30196af59bb0957 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 04:02:12 +0000 Subject: [PATCH 2/7] Fix reader release E2E failures --- cmd/playground-chaos/reset.go | 18 +++++++ internal/controller/updater.go | 10 ++++ internal/controller/updater_test.go | 11 ++++- .../scenarios/s31_pitr_verification_rustfs.go | 47 ++++++++++++++++--- .../scenarios/s40_reader_data_loss.go | 13 ++++- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/cmd/playground-chaos/reset.go b/cmd/playground-chaos/reset.go index 9f727ef..db906f1 100644 --- a/cmd/playground-chaos/reset.go +++ b/cmd/playground-chaos/reset.go @@ -114,6 +114,7 @@ func (r *resetter) run(ctx context.Context) error { {"create replication users", r.createReplicationUsers}, {"start operator", r.startOperator}, {"wait healthy baseline", r.waitBaseline}, + {"normalize fresh baseline", r.normalizeFreshBaseline}, } for _, phase := range phases { r.info("%s...", phase.name) @@ -125,6 +126,23 @@ func (r *resetter) run(ctx context.Context) error { return nil } +// normalizeFreshBaseline removes the failover stamp produced while the +// operator resolves the all-writable shape of newly initialized MySQL pods. +// Restarting after the cluster has converged lets the manager discover the +// existing primary without recording bootstrap as an operational failover. +func (r *resetter) normalizeFreshBaseline(ctx context.Context, sites []v1alpha1.SiteSpec) error { + if err := r.scaleOperatorDown(ctx, sites); err != nil { + return err + } + if err := r.clearMFGStatus(ctx, sites); err != nil { + return err + } + if err := r.startOperator(ctx, sites); err != nil { + return err + } + return r.waitBaseline(ctx, sites) +} + func (r *resetter) scaleOperatorDown(ctx context.Context, _ []v1alpha1.SiteSpec) error { if err := r.scaleDeploymentIfExists(ctx, operatorDeployment, 0); err != nil { return err diff --git a/internal/controller/updater.go b/internal/controller/updater.go index e298b47..668367e 100644 --- a/internal/controller/updater.go +++ b/internal/controller/updater.go @@ -275,6 +275,16 @@ func (u *UpdateController) waitForReplicaReadyExpected(ctx context.Context, chec deadline := time.Now().Add(timeout) for { readOnly, roleErr := checker.CheckReadOnly(ctx) + if roleErr == nil && !readOnly { + // MySQL does not persist SET GLOBAL read_only across a pod + // replacement. Re-fence the updated follower before restarting + // its retained replication configuration. + if err := checker.SetSuperReadOnly(ctx, true); err == nil { + if err := checker.SetReadOnly(ctx, true); err == nil { + readOnly = true + } + } + } rs, err := checker.ShowReplicaStatus(ctx) if roleErr == nil && readOnly && err == nil && rs != nil { if canonicalSourceHost(rs.SourceHost) == canonicalSourceHost(expected) && (!rs.IORunning || !rs.SQLRunning) { diff --git a/internal/controller/updater_test.go b/internal/controller/updater_test.go index de9d84c..5e69aec 100644 --- a/internal/controller/updater_test.go +++ b/internal/controller/updater_test.go @@ -168,11 +168,20 @@ func TestUpdateController_ExecuteTargetsActiveLastRequiresDirectReplica(t *testi } func TestWaitForReplicaReadyExpectedRequiresReadOnlyDirectThreads(t *testing.T) { + writable := &testutil.FakeMySQL{ReadOnlyVal: false, ReplicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "new-primary"}} + uc := NewUpdateController(NewFailoverController(testutil.TestLogger()), testutil.TestLogger()) + uc.tickInterval = time.Millisecond + if err := uc.waitForReplicaReadyExpected(context.Background(), writable, "new-primary", 10*time.Millisecond); err != nil { + t.Fatalf("updated writable follower was not recovered: %v", err) + } + if !writable.SuperReadOnlySet || !writable.ReadOnlyVal { + t.Fatal("updated writable follower was not re-fenced") + } + for _, tc := range []struct { name string checker *testutil.FakeMySQL }{ - {name: "writable", checker: &testutil.FakeMySQL{ReadOnlyVal: false, ReplicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "new-primary"}}}, {name: "stopped threads", checker: &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{SourceHost: "new-primary"}}}, {name: "wrong source", checker: &testutil.FakeMySQL{ReadOnlyVal: true, ReplicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "old-primary"}}}, } { diff --git a/internal/playground/scenarios/s31_pitr_verification_rustfs.go b/internal/playground/scenarios/s31_pitr_verification_rustfs.go index 418f1b1..6913534 100644 --- a/internal/playground/scenarios/s31_pitr_verification_rustfs.go +++ b/internal/playground/scenarios/s31_pitr_verification_rustfs.go @@ -91,7 +91,7 @@ func s31WaitArchiverReady() runner.Step { Do: func(ctx context.Context, env *runner.Env) error { waitCtx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer cancel() - mfg, err := env.Wait.UntilCR(waitCtx, env.Namespace, "PITR spec rollout", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + _, err := env.Wait.UntilCR(waitCtx, env.Namespace, "PITR spec rollout", func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { ready := false for _, c := range mfg.Status.Conditions { if c.Type == "Ready" { @@ -104,18 +104,17 @@ func s31WaitArchiverReady() runner.Step { if err != nil { return err } - active := mfg.Status.ActiveSite if _, err := env.Logs("operator"); err != nil { env.Capture.Note("open operator logs failed: " + err.Error()) } - if _, err := env.Logs("sidecar:" + active); err != nil { - env.Capture.Note("open sidecar logs failed: " + err.Error()) - } wantPrefix := path.Join(ctxFetch(env, "backupPrefix"), "binlogs") - st, err := waitForArchiverReady(waitCtx, env, active, wantPrefix) + active, st, err := waitForActiveArchiverReady(waitCtx, env, wantPrefix) if err != nil { return err } + if _, err := env.Logs("sidecar:" + active); err != nil { + env.Capture.Note("open sidecar logs failed: " + err.Error()) + } env.Capture.Note("active archiver ready: " + archiverSummary(st)) if _, replica, err := activeAndReplica(ctx, env); err == nil { if passive, err := env.Sidecar(replica); err == nil { @@ -129,6 +128,42 @@ func s31WaitArchiverReady() runner.Step { } } +func waitForActiveArchiverReady(ctx context.Context, env *runner.Env, wantPrefix string) (string, *pgsidecar.ArchiverStatusResponse, error) { + tick := time.NewTicker(time.Second) + defer tick.Stop() + var last string + for { + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + last = err.Error() + } else if mfg.Status.ActiveSite == "" { + last = "active site is empty" + } else { + active := mfg.Status.ActiveSite + probe, err := pgsidecar.Open(ctx, env.Kube, env.Namespace, env.FG, active) + if err != nil { + last = fmt.Sprintf("active=%s: %v", active, err) + } else { + st, statusErr := probe.ArchiverStatus(ctx) + probe.Close() + if statusErr != nil { + last = fmt.Sprintf("active=%s: %v", active, statusErr) + } else { + last = fmt.Sprintf("active=%s %s", active, archiverSummary(st)) + if st.Enabled && st.Primary && st.StorageType == string(v1alpha1.BackupStorageS3) && strings.Trim(st.ManifestPrefix, "/") == strings.Trim(wantPrefix, "/") && st.LastError == "" && !st.LastScanAt.IsZero() { + return active, st, nil + } + } + } + } + select { + case <-ctx.Done(): + return "", nil, fmt.Errorf("wait for active archiver readiness: %w (last: %s)", ctx.Err(), last) + case <-tick.C: + } + } +} + func waitForArchiverReady(ctx context.Context, env *runner.Env, site, wantPrefix string) (*pgsidecar.ArchiverStatusResponse, error) { tick := time.NewTicker(time.Second) defer tick.Stop() diff --git a/internal/playground/scenarios/s40_reader_data_loss.go b/internal/playground/scenarios/s40_reader_data_loss.go index 03f16f2..da31f26 100644 --- a/internal/playground/scenarios/s40_reader_data_loss.go +++ b/internal/playground/scenarios/s40_reader_data_loss.go @@ -136,8 +136,17 @@ func s40SeedAndReplicateMarker(state *s40RunState) runner.Step { return fmt.Errorf("open active MySQL %s: %w", state.active, err) } defer active.Close() - if _, err := active.Exec(ctx, "CREATE DATABASE IF NOT EXISTS bloodraven_reader_e2e; CREATE TABLE IF NOT EXISTS "+s40MarkerTable+" (marker VARCHAR(128) PRIMARY KEY); INSERT INTO "+s40MarkerTable+" (marker) VALUES (?)", state.marker); err != nil { - return fmt.Errorf("write marker %q: %w", state.marker, err) + for _, stmt := range []struct { + query string + args []any + }{ + {query: "CREATE DATABASE IF NOT EXISTS bloodraven_reader_e2e"}, + {query: "CREATE TABLE IF NOT EXISTS " + s40MarkerTable + " (marker VARCHAR(128) PRIMARY KEY)"}, + {query: "INSERT INTO " + s40MarkerTable + " (marker) VALUES (?)", args: []any{state.marker}}, + } { + if _, err := active.Exec(ctx, stmt.query, stmt.args...); err != nil { + return fmt.Errorf("write marker %q: %w", state.marker, err) + } } return s40WaitForMarker(ctx, env, state.reader, state.marker, 60*time.Second) }, From d8d689ca7440350af0b75785b678bd31023b0a06 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 05:09:04 +0000 Subject: [PATCH 3/7] Fix reader endpoint safety assertion --- .../playground/scenarios/s40_observer_test.go | 41 +++++++++++++++++++ .../scenarios/s40_reader_data_loss.go | 10 ++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/internal/playground/scenarios/s40_observer_test.go b/internal/playground/scenarios/s40_observer_test.go index 583879c..6b73ecc 100644 --- a/internal/playground/scenarios/s40_observer_test.go +++ b/internal/playground/scenarios/s40_observer_test.go @@ -117,6 +117,47 @@ func TestS40ObserveOnceAllowsExpectedEmptyListTransitions(t *testing.T) { } } +func TestS40ObserveOnceIgnoresTerminatingServingEndpoint(t *testing.T) { + ready := false + serving := true + terminating := true + portName := "mysql" + port := int32(3306) + clientset := k8sfake.NewSimpleClientset(&discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reader-client", + Namespace: pgkube.PlaygroundNamespace, + Labels: map[string]string{discoveryv1.LabelServiceName: pgkube.MysqlDeploymentName(pgkube.FailoverGroupName, "reader")}, + }, + Ports: []discoveryv1.EndpointPort{{Name: &portName, Port: &port}}, + Endpoints: []discoveryv1.Endpoint{{ + Conditions: discoveryv1.EndpointConditions{Ready: &ready, Serving: &serving, Terminating: &terminating}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "reader-old", UID: types.UID("original-pod")}, + }}, + }) + env, state := s40ObserverTestEnv(t, clientset) + mfg, err := env.Kube.GetMFGNamed(context.Background(), env.Namespace, env.FG) + if err != nil { + t.Fatal(err) + } + s40StatusSite(mfg, "reader").Replicating = false + if err := env.Kube.Controller.Update(context.Background(), mfg); err != nil { + t.Fatal(err) + } + observation := &s40ContinuousObservation{} + + s40ObserveOnce(context.Background(), env, state, observation) + + observation.mu.Lock() + defer observation.mu.Unlock() + if observation.err != nil { + t.Fatalf("terminating drain endpoint recorded as client-routable: %v", observation.err) + } + if !observation.sawClientEndpointEmpty { + t.Fatal("ready=false terminating endpoint did not record client endpoint absence") + } +} + func TestS40ObserveReadErrorIgnoresOnlyObserverCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/internal/playground/scenarios/s40_reader_data_loss.go b/internal/playground/scenarios/s40_reader_data_loss.go index da31f26..d31fa87 100644 --- a/internal/playground/scenarios/s40_reader_data_loss.go +++ b/internal/playground/scenarios/s40_reader_data_loss.go @@ -301,14 +301,14 @@ func s40ObserveOnce(ctx context.Context, env *runner.Env, state *s40RunState, ob clientState, clientErr := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.reader)) s40ObserveReadError(ctx, observation, "read reader client EndpointSlices", clientErr) if clientErr == nil { - serving := clientState.ServingPodNames("mysql") - if len(serving) == 0 { + ready := clientState.ReadyPodNames("mysql") + if len(ready) == 0 { // Empty EndpointSlice results are expected while scale-to-zero or // endpoint turnover is in progress. The successful list, rather than // an ignored NotFound/read error, is the explicit absence signal. observation.markClientEmpty() } else if mfgErr == nil && !readerServingHealthy { - observation.fail(fmt.Errorf("reader client Service published serving pods %v while reader status was unhealthy", serving)) + observation.fail(fmt.Errorf("reader client Service published ready pods %v while reader status was unhealthy", ready)) } } @@ -323,8 +323,8 @@ func s40ObserveOnce(ctx context.Context, env *runner.Env, state *s40RunState, ob if string(pod.UID) == state.originalPod || s40PodReady(pod) { continue } - if clientErr == nil && len(clientState.ServingPodNames("mysql")) != 0 { - observation.fail(fmt.Errorf("reader client Service published a serving endpoint while replacement pod %s was not ready", pod.Name)) + if clientErr == nil && len(clientState.ReadyPodNames("mysql")) != 0 { + observation.fail(fmt.Errorf("reader client Service published a ready endpoint while replacement pod %s was not ready", pod.Name)) } internal, internalErr := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.reader)+"-internal") s40ObserveReadError(ctx, observation, "read reader internal EndpointSlices", internalErr) From 873e6d252c4e4d413c663dbefce5feecad9f9b8c Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 05:44:22 +0000 Subject: [PATCH 4/7] Add reader chaos scenarios 41-44 from issue #115 proposals Automates the R3/R4/R5/R9 chaos proposals from the issue #115 review comment as playground scenarios with the standard Precheck/forensics contract: - 41-reader-availability-during-failover (R3, release): the reader answers SELECTs continuously through an unplanned failover and then repoints directly at the new primary; source-host history proves no chained or blocked intermediate state. - 42-reader-stall-no-group-degradation (R4, smoke+release): SOURCE_DELAY grows reader lag past readOnlyMaxLagSeconds for a 3x-maxLagSeconds soak with zero group-level effect; only endpoint shedding plus lag and source-state metrics react. - 43-writable-reader-fence (R5, release): a writable reader is fenced without debounce, a planned failover targeting it is rejected with the role error, and its errant GTID trips the convergence containment gate into Blocked/GTIDDiverged; cleanup reconciles the errant transactions as empty transactions on the primary. - 44-reader-source-convergence-invariant (R9, release): a manually repointed (chained) reader heals back to the direct primary as a poll-loop invariant, emitting the documented convergence log events with no failover consumed. Shared reader helpers (topology resolution, serving-status contract, marker seeding/replication, endpoint waits) move out of scenario 40 into reader_helpers.go for reuse. R6-R8 remain documented as manual follow-ups in chaos-scenarios.md. --- docs/docs/playground.mdx | 8 +- internal/playground/runner/profile.go | 53 +- .../runner/profile_registry_test.go | 27 +- .../playground/scenarios/inventory_40_test.go | 8 +- .../scenarios/inventory_41_44_test.go | 123 +++++ .../playground/scenarios/reader_helpers.go | 231 +++++++++ .../playground/scenarios/s40_observer_test.go | 2 +- .../scenarios/s40_reader_data_loss.go | 99 +--- ...s41_reader_availability_during_failover.go | 323 ++++++++++++ .../scenarios/s42_reader_stall_isolation.go | 301 +++++++++++ .../scenarios/s43_writable_reader_fence.go | 489 ++++++++++++++++++ ...s44_reader_source_convergence_invariant.go | 237 +++++++++ playground/chaos-scenarios.md | 91 ++++ 13 files changed, 1866 insertions(+), 126 deletions(-) create mode 100644 internal/playground/scenarios/inventory_41_44_test.go create mode 100644 internal/playground/scenarios/reader_helpers.go create mode 100644 internal/playground/scenarios/s41_reader_availability_during_failover.go create mode 100644 internal/playground/scenarios/s42_reader_stall_isolation.go create mode 100644 internal/playground/scenarios/s43_writable_reader_fence.go create mode 100644 internal/playground/scenarios/s44_reader_source_convergence_invariant.go diff --git a/docs/docs/playground.mdx b/docs/docs/playground.mdx index 2b1e151..63089e8 100644 --- a/docs/docs/playground.mdx +++ b/docs/docs/playground.mdx @@ -213,7 +213,13 @@ make chaos-run-all-profile PROFILE=smoke # Core failover subset (~5 minu make chaos-run-all # The full suite ``` -Scenario 40 deterministically interprets reader node/storage loss as Deployment scale-down plus PVC replacement on the dedicated reader worker. It continuously proves reader loss does not unset group `Ready`, verifies the reader client endpoint is shed during clone/catch-up, and confirms direct-source replication and endpoint return after auto-clone. +Scenarios 40-44 cover the dedicated read-only reader site end to end: + +- **40** deterministically interprets reader node/storage loss as Deployment scale-down plus PVC replacement on the dedicated reader worker. It continuously proves reader loss does not unset group `Ready`, verifies the reader client endpoint is shed during clone/catch-up, and confirms direct-source replication and endpoint return after auto-clone. +- **41** answers reader `SELECT`s continuously through an unplanned failover (staleness allowed, availability required) and asserts the reader repoints directly to the new primary with no chained or blocked intermediate state. +- **42** (smoke profile) stalls only the reader with `SOURCE_DELAY` and proves unbounded reader lag has zero group-level effect: no failover, no cooldown consumed, just endpoint shedding plus alertable status and metrics. +- **43** makes the reader anomalously writable with an errant row: the operator fences it without debounce, a planned failover targeting it is rejected with the role error, and the errant GTID blocks source convergence at the containment gate instead of silently repointing. +- **44** manually repoints the reader at the standby and proves direct-source convergence is a poll-loop invariant — any wrong-source state heals, with the documented convergence log events and no failover. This same suite is Bloodraven's E2E test bed in CI: the full suite runs nightly against a real Kubernetes cluster, and the smoke profile gates diff --git a/internal/playground/runner/profile.go b/internal/playground/runner/profile.go index 11b3186..8ca7a11 100644 --- a/internal/playground/runner/profile.go +++ b/internal/playground/runner/profile.go @@ -4,13 +4,17 @@ package runner // form a strict superset chain: smoke ⊂ release ⊂ full. // // - smoke: short PR-label/manual subset covering emergency failover, -// planned switchover, and operator restart durability (~3 scenarios). +// planned switchover, operator restart durability, and reader-stall +// isolation (~4 scenarios). // - release: curated release/nightly subset covering the WISHLIST #32/#43 // behaviours (emergency failover, planned switchover, operator restart, // data integrity, operator kill during failover, self-fencing, // network partition, PVC loss/re-bootstrap, old-primary recovery, -// failover state durability, backup verification, PITR verification, -// reader PVC loss, and direct-source auto-clone). +// failover state durability, backup verification, PITR verification) +// plus the issue #115 reader-site behaviours (reader PVC loss and +// direct-source auto-clone, reader availability through unplanned +// failover, reader stall isolation, writable-reader fencing, and the +// source-convergence invariant). // - full: every registered scenario (existing run-all behaviour). type Profile string @@ -23,14 +27,15 @@ const ( // DefaultProfile is used when --profile is not supplied. const DefaultProfile Profile = ProfileFull -// smokeScenarios is the hard-coded smoke subset. These three scenarios +// smokeScenarios is the hard-coded smoke subset. These four scenarios // exercise the critical path — emergency failover, planned switchover, -// operator restart — and complete in roughly 3-5 minutes on a warm -// playground cluster. +// operator restart, reader-stall isolation — and complete in roughly +// 5-8 minutes on a warm playground cluster. var smokeScenarios = map[string]bool{ - "01-clean-primary-kill": true, // emergency failover - "02-planned-switchover": true, // planned switchover - "02-operator-kill-restart": true, // operator restart durability + "01-clean-primary-kill": true, // emergency failover + "02-planned-switchover": true, // planned switchover (includes reader repoint assertions) + "02-operator-kill-restart": true, // operator restart durability + "42-reader-stall-no-group-degradation": true, // reader stall isolation (issue #115 R4; fast, no clone wait) } // releaseScenarios is the hard-coded release/nightly subset. In addition @@ -42,20 +47,24 @@ var smokeScenarios = map[string]bool{ // backup/PITR verification coverage. var releaseScenarios = map[string]bool{ // smoke scenarios (superset) - "01-clean-primary-kill": true, - "02-planned-switchover": true, - "02-operator-kill-restart": true, + "01-clean-primary-kill": true, + "02-planned-switchover": true, + "02-operator-kill-restart": true, + "42-reader-stall-no-group-degradation": true, // additional release scenarios - "04-data-integrity-on-failover": true, // data plane correctness - "05-operator-kill-during-failover": true, // operator resilience mid-failover - "06-self-fence-isolated-primary": true, // taint/DNS self-fencing - "09-network-partition-self-fence": true, // NetworkPolicy/partition - "10-full-bootstrap-after-data-wipe": true, // PVC loss → re-bootstrap - "12-old-primary-recovery-no-divergence": true, // old-primary recovery - "23-failover-state-durability": true, // state survives operator restart - "30-backup-verification-rustfs": true, // RustFS backup restore verification - "31-pitr-verification-rustfs": true, // RustFS PITR replay verification - "40-reader-data-loss-reclone": true, // reader PVC loss, endpoint shedding, and direct-source auto-clone + "04-data-integrity-on-failover": true, // data plane correctness + "05-operator-kill-during-failover": true, // operator resilience mid-failover + "06-self-fence-isolated-primary": true, // taint/DNS self-fencing + "09-network-partition-self-fence": true, // NetworkPolicy/partition + "10-full-bootstrap-after-data-wipe": true, // PVC loss → re-bootstrap + "12-old-primary-recovery-no-divergence": true, // old-primary recovery + "23-failover-state-durability": true, // state survives operator restart + "30-backup-verification-rustfs": true, // RustFS backup restore verification + "31-pitr-verification-rustfs": true, // RustFS PITR replay verification + "40-reader-data-loss-reclone": true, // reader PVC loss, endpoint shedding, and direct-source auto-clone + "41-reader-availability-during-failover": true, // reader keeps serving through unplanned failover, then repoints + "43-writable-reader-fence": true, // writable reader fenced, rejected as target, blocked on divergence + "44-reader-source-convergence-invariant": true, // wrong-source reader heals as a poll-loop invariant } // Profiles returns the list of valid profile names for CLI help and diff --git a/internal/playground/runner/profile_registry_test.go b/internal/playground/runner/profile_registry_test.go index d84c5a6..d289ad4 100644 --- a/internal/playground/runner/profile_registry_test.go +++ b/internal/playground/runner/profile_registry_test.go @@ -14,17 +14,20 @@ func TestProfilesSelectRegisteredScenarios(t *testing.T) { } smoke := runner.SelectForProfile(all, runner.ProfileSmoke) - if len(smoke) != 3 { - t.Fatalf("smoke profile selected %d scenarios, want 3", len(smoke)) + if len(smoke) != 4 { + t.Fatalf("smoke profile selected %d scenarios, want 4", len(smoke)) + } + if !containsScenarioID(smoke, "42-reader-stall-no-group-degradation") { + t.Error("smoke profile must include 42-reader-stall-no-group-degradation (issue #115 R4)") } - // The release subset lists 13 members, all of which are now selected: + // The release subset lists 17 members, all of which are now selected: // 31-pitr-verification-rustfs is no longer quarantined (#101 fixed — the // verify mysqld runs gtid_mode=ON and server-side dedup handles the PITR - // replay). + // replay), and the issue #115 reader scenarios (40-44) are release-graded. release := runner.SelectForProfile(all, runner.ProfileRelease) - if len(release) != 13 { - t.Fatalf("release profile selected %d scenarios, want 13", len(release)) + if len(release) != 17 { + t.Fatalf("release profile selected %d scenarios, want 17", len(release)) } if !containsScenarioID(release, "09-network-partition-self-fence") { t.Error("release profile must include 09-network-partition-self-fence (no longer quarantined)") @@ -32,8 +35,16 @@ func TestProfilesSelectRegisteredScenarios(t *testing.T) { if !containsScenarioID(release, "31-pitr-verification-rustfs") { t.Error("31-pitr-verification-rustfs is no longer quarantined (#101) and must be in the release profile") } - if !containsScenarioID(release, "40-reader-data-loss-reclone") { - t.Error("40-reader-data-loss-reclone must be in the release profile") + for _, id := range []string{ + "40-reader-data-loss-reclone", + "41-reader-availability-during-failover", + "42-reader-stall-no-group-degradation", + "43-writable-reader-fence", + "44-reader-source-convergence-invariant", + } { + if !containsScenarioID(release, id) { + t.Errorf("%s must be in the release profile", id) + } } // full = every registered scenario minus any that are quarantined. Computed diff --git a/internal/playground/scenarios/inventory_40_test.go b/internal/playground/scenarios/inventory_40_test.go index 6728167..8716e2f 100644 --- a/internal/playground/scenarios/inventory_40_test.go +++ b/internal/playground/scenarios/inventory_40_test.go @@ -39,7 +39,7 @@ func TestCanonicalMySQLHost(t *testing.T) { } } -func TestS40AssertReaderStatus(t *testing.T) { +func TestAssertReaderServingStatus(t *testing.T) { maxLag := int64(10) lag := int64(10) mfg := &v1alpha1.MysqlFailoverGroup{ @@ -55,16 +55,16 @@ func TestS40AssertReaderStatus(t *testing.T) { SourceHost: "mysql-playground-iad-internal.ns.svc.cluster.local:3306", SourceConvergenceState: v1alpha1.SourceConvergenceConverged, } - if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err != nil { + if err := assertReaderServingStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err != nil { t.Fatalf("exact-threshold reader rejected: %v", err) } lag = 11 - if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "exceeds") { + if err := assertReaderServingStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "exceeds") { t.Fatalf("over-threshold error = %v, want exceeds", err) } status.SecondsBehindSource = nil - if err := s40AssertReaderStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "unknown") { + if err := assertReaderServingStatus(mfg, &status, "mysql-playground-iad-internal.ns.svc.cluster.local"); err == nil || !strings.Contains(err.Error(), "unknown") { t.Fatalf("unknown-lag error = %v, want unknown", err) } } diff --git a/internal/playground/scenarios/inventory_41_44_test.go b/internal/playground/scenarios/inventory_41_44_test.go new file mode 100644 index 0000000..a9c8eb1 --- /dev/null +++ b/internal/playground/scenarios/inventory_41_44_test.go @@ -0,0 +1,123 @@ +package scenarios + +import ( + "strings" + "testing" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +func TestReaderScenarios41To44RegisteredAndProfiled(t *testing.T) { + releaseIDs := []string{ + "41-reader-availability-during-failover", + "42-reader-stall-no-group-degradation", + "43-writable-reader-fence", + "44-reader-source-convergence-invariant", + } + release := runner.SelectForProfile(runner.DefaultRegistry.List(), runner.ProfileRelease) + smoke := runner.SelectForProfile(runner.DefaultRegistry.List(), runner.ProfileSmoke) + for _, id := range releaseIDs { + scenario, ok := runner.DefaultRegistry.Get(id) + if !ok { + t.Errorf("scenario %s is not registered", id) + continue + } + if scenario.Quarantine != "" { + t.Errorf("scenario %s must not be quarantined: %q", id, scenario.Quarantine) + } + if scenario.Timeout <= 0 || scenario.Precheck == nil || len(scenario.Steps) == 0 { + t.Errorf("scenario %s is incomplete: timeout=%s precheck=%v steps=%d", id, scenario.Timeout, scenario.Precheck != nil, len(scenario.Steps)) + } + if !inventoryContains(release, id) { + t.Errorf("scenario %s is not selected by the release profile", id) + } + } + if !inventoryContains(smoke, "42-reader-stall-no-group-degradation") { + t.Error("42-reader-stall-no-group-degradation must be in the smoke profile (issue #115 R4)") + } +} + +func TestFindReadOnlyReaderSite(t *testing.T) { + readOnly := v1alpha1.SiteRoleReadOnly + candidate := v1alpha1.SiteRolePrimaryCandidate + mfg := &v1alpha1.MysqlFailoverGroup{Spec: v1alpha1.MysqlFailoverGroupSpec{Sites: []v1alpha1.SiteSpec{ + {Name: "iad", Role: candidate}, + {Name: "pdx", Role: candidate}, + }}} + if _, err := findReadOnlyReaderSite(mfg); err == nil { + t.Error("expected error with zero read-only sites") + } + mfg.Spec.Sites = append(mfg.Spec.Sites, v1alpha1.SiteSpec{Name: "reader", Role: readOnly}) + reader, err := findReadOnlyReaderSite(mfg) + if err != nil || reader != "reader" { + t.Errorf("findReadOnlyReaderSite = %q, %v; want reader, nil", reader, err) + } + mfg.Spec.Sites = append(mfg.Spec.Sites, v1alpha1.SiteSpec{Name: "reader2", Role: readOnly}) + if _, err := findReadOnlyReaderSite(mfg); err == nil { + t.Error("expected error with two read-only sites") + } +} + +func TestGtidSetTransactions(t *testing.T) { + got, err := gtidSetTransactions("aaaa:1-3:7") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := []string{"aaaa:1", "aaaa:2", "aaaa:3", "aaaa:7"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } + + if _, err := gtidSetTransactions("aaaa:1-2,bbbb:5"); err != nil { + t.Errorf("multi-uuid set should parse: %v", err) + } + for _, bad := range []string{"", "no-intervals", "aaaa:x", "aaaa:5-2", "aaaa:1-100"} { + if _, err := gtidSetTransactions(bad); err == nil { + t.Errorf("gtidSetTransactions(%q) succeeded, want error", bad) + } + } +} + +func TestS41ObservationResult(t *testing.T) { + state := &s41RunState{ + oldHost: "mysql-playground-iad-internal.ns.svc.cluster.local", + newHost: "mysql-playground-pdx-internal.ns.svc.cluster.local", + } + healthy := func() *s41Observation { + return &s41Observation{reads: 40, sourceHosts: []string{state.oldHost, state.newHost}} + } + + if err := healthy().result(state); err != nil { + t.Errorf("clean old->new history rejected: %v", err) + } + + o := healthy() + o.reads = 5 + if err := o.result(state); err == nil || !strings.Contains(err.Error(), "too few") { + t.Errorf("too-few-reads error = %v", err) + } + + o = healthy() + o.sourceHosts = []string{state.oldHost} + if err := o.result(state); err == nil || !strings.Contains(err.Error(), "never reported the new primary") { + t.Errorf("never-new error = %v", err) + } + + o = healthy() + o.sourceHosts = []string{state.oldHost, state.newHost, state.oldHost} + if err := o.result(state); err == nil || !strings.Contains(err.Error(), "flipped back") { + t.Errorf("flip-back error = %v", err) + } + + o = healthy() + o.sourceHosts = []string{state.oldHost, "mysql-playground-other-internal.ns.svc.cluster.local", state.newHost} + if err := o.result(state); err == nil || !strings.Contains(err.Error(), "unexpected replication source") { + t.Errorf("foreign-host error = %v", err) + } +} diff --git a/internal/playground/scenarios/reader_helpers.go b/internal/playground/scenarios/reader_helpers.go new file mode 100644 index 0000000..3fb0779 --- /dev/null +++ b/internal/playground/scenarios/reader_helpers.go @@ -0,0 +1,231 @@ +package scenarios + +import ( + "context" + "fmt" + "strings" + "time" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + pgmysql "github.com/shipstream/bloodraven/internal/playground/mysql" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +// Shared helpers for the reader-site scenarios (40-44). All of them +// operate on the playground's 3-site topology: two primary-candidates +// plus exactly one dedicated read-only reader. + +// statusSiteByName returns the status entry for the named site, or nil. +func statusSiteByName(mfg *v1alpha1.MysqlFailoverGroup, name string) *v1alpha1.SiteStatus { + for i := range mfg.Status.Sites { + if mfg.Status.Sites[i].Name == name { + return &mfg.Status.Sites[i] + } + } + return nil +} + +// playgroundInternalSiteHost is the per-site internal Service FQDN the +// operator uses as the replication source host. +func playgroundInternalSiteHost(group, site, namespace string) string { + return fmt.Sprintf("mysql-%s-%s-internal.%s.svc.cluster.local", group, site, namespace) +} + +// canonicalMySQLHost normalizes a MySQL host string for comparison: +// lower-cased, trimmed, without a :3306 suffix or trailing dot. +func canonicalMySQLHost(host string) string { + host = strings.ToLower(strings.TrimSpace(host)) + host = strings.TrimSuffix(host, ":3306") + return strings.TrimSuffix(host, ".") +} + +// assertReaderServingStatus asserts the CR status shape of a healthy, +// serving reader: read-only state, replicating, converged directly onto +// expectedHost, with known lag at or below readOnlyMaxLagSeconds. +func assertReaderServingStatus(mfg *v1alpha1.MysqlFailoverGroup, status *v1alpha1.SiteStatus, expectedHost string) error { + if status.State != "read-only" { + return fmt.Errorf("state=%q, want read-only", status.State) + } + if !status.Replicating { + return fmt.Errorf("replicating=false") + } + if status.SourceConvergenceState != v1alpha1.SourceConvergenceConverged { + return fmt.Errorf("sourceConvergenceState=%q, want Converged", status.SourceConvergenceState) + } + if canonicalMySQLHost(status.SourceHost) != canonicalMySQLHost(expectedHost) { + return fmt.Errorf("sourceHost=%q, want %q", status.SourceHost, expectedHost) + } + if status.SecondsBehindSource == nil { + return fmt.Errorf("secondsBehindSource is unknown") + } + maxLag := mfg.Spec.EffectiveReadOnlyMaxLagSeconds() + if *status.SecondsBehindSource > maxLag { + return fmt.Errorf("secondsBehindSource=%d exceeds reader threshold %d", *status.SecondsBehindSource, maxLag) + } + return nil +} + +// findReadOnlyReaderSite returns the name of the single read-only role +// site, erroring when zero or multiple readers are present. +func findReadOnlyReaderSite(mfg *v1alpha1.MysqlFailoverGroup) (string, error) { + reader := "" + for i := range mfg.Spec.Sites { + if mfg.Spec.Sites[i].IsReadOnlyReader() { + if reader != "" { + return "", fmt.Errorf("requires exactly one read-only role site, found at least %q and %q", reader, mfg.Spec.Sites[i].Name) + } + reader = mfg.Spec.Sites[i].Name + } + } + if reader == "" { + return "", fmt.Errorf("requires exactly one read-only role site") + } + return reader, nil +} + +// readerTopology names the three roles of the playground topology at +// scenario start plus the active site's internal replication host. +type readerTopology struct { + reader string + active string + standby string + activeHost string +} + +// resolveReaderTopology runs the shared healthy-baseline precheck and +// then resolves and validates the reader topology: exactly one reader, +// a promotable active site, a promotable standby, a serving reader +// status converged on the active site, and the reader's client Service +// publishing exactly the reader pod. +func resolveReaderTopology(ctx context.Context, env *runner.Env) (readerTopology, error) { + var topo readerTopology + if err := AssertHealthyBaseline(ctx, env); err != nil { + return topo, err + } + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return topo, err + } + topo.reader, err = findReadOnlyReaderSite(mfg) + if err != nil { + return topo, fmt.Errorf("precheck: %w", err) + } + topo.active = mfg.Status.ActiveSite + if site := mfg.Spec.SiteByName(topo.active); site == nil || !site.IsPromotable() { + return topo, fmt.Errorf("active site %q is missing or non-promotable", topo.active) + } + topo.standby, err = PeerOf(mfg, topo.active) + if err != nil { + return topo, err + } + topo.activeHost = playgroundInternalSiteHost(env.FG, topo.active, env.Namespace) + + readerStatus := statusSiteByName(mfg, topo.reader) + if readerStatus == nil { + return topo, fmt.Errorf("reader %q missing from status", topo.reader) + } + if err := assertReaderServingStatus(mfg, readerStatus, topo.activeHost); err != nil { + return topo, fmt.Errorf("reader status precheck: %w", err) + } + + pod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, topo.reader) + if err != nil { + return topo, err + } + endpoints, err := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, topo.reader)) + if err != nil { + return topo, fmt.Errorf("read reader client EndpointSlices: %w", err) + } + ready := endpoints.ReadyPodNames("mysql") + if len(ready) != 1 || ready[0] != pod.Name { + return topo, fmt.Errorf("reader client Service ready pods=%v, want exactly %s", ready, pod.Name) + } + return topo, nil +} + +// waitReaderClientEndpoint polls the reader's client Service until its +// EndpointSlices publish exactly the reader's current MySQL pod as +// ready on the mysql port. +func waitReaderClientEndpoint(ctx context.Context, env *runner.Env, reader string, timeout time.Duration) error { + pod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, reader) + if err != nil { + return err + } + deadline := time.Now().Add(timeout) + var last string + for time.Now().Before(deadline) { + endpoints, err := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, reader)) + if err != nil { + last = err.Error() + } else { + ready := endpoints.ReadyPodNames("mysql") + last = fmt.Sprintf("ready=%v", ready) + if len(ready) == 1 && ready[0] == pod.Name { + return nil + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("reader client endpoint did not publish pod %s within %s (last: %s)", pod.Name, timeout, last) +} + +// seedMarkerRow creates the marker schema on the given site (normally +// the active primary) and inserts one marker row. qualifiedTable must be +// a `db.table` name; the database and table are created on demand. +func seedMarkerRow(ctx context.Context, env *runner.Env, site, qualifiedTable, marker string) error { + dbName, _, ok := strings.Cut(qualifiedTable, ".") + if !ok { + return fmt.Errorf("qualified table %q must be db.table", qualifiedTable) + } + client, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, site, env.Creds) + if err != nil { + return fmt.Errorf("open MySQL %s: %w", site, err) + } + defer client.Close() + // Statements run one at a time: go-sql-driver rejects placeholder + // args inside a multi-statement batch. + for _, stmt := range []struct { + query string + args []any + }{ + {query: "CREATE DATABASE IF NOT EXISTS " + dbName}, + {query: "CREATE TABLE IF NOT EXISTS " + qualifiedTable + " (marker VARCHAR(128) PRIMARY KEY)"}, + {query: "INSERT INTO " + qualifiedTable + " (marker) VALUES (?)", args: []any{marker}}, + } { + if _, err := client.Exec(ctx, stmt.query, stmt.args...); err != nil { + return fmt.Errorf("write marker %q on %s: %w", marker, site, err) + } + } + return nil +} + +// waitForMarkerOnSite polls the named site until the marker row is +// visible, proving live end-to-end replication into that site. +func waitForMarkerOnSite(ctx context.Context, env *runner.Env, site, qualifiedTable, marker string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + client, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, site, env.Creds) + if err == nil { + count, queryErr := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+qualifiedTable+" WHERE marker=?", marker) + _ = client.Close() + if queryErr == nil && count == 1 { + return nil + } + last = queryErr + } else { + last = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("marker %q did not replicate to %s within %s: %v", marker, site, timeout, last) +} diff --git a/internal/playground/scenarios/s40_observer_test.go b/internal/playground/scenarios/s40_observer_test.go index 6b73ecc..c963f75 100644 --- a/internal/playground/scenarios/s40_observer_test.go +++ b/internal/playground/scenarios/s40_observer_test.go @@ -140,7 +140,7 @@ func TestS40ObserveOnceIgnoresTerminatingServingEndpoint(t *testing.T) { if err != nil { t.Fatal(err) } - s40StatusSite(mfg, "reader").Replicating = false + statusSiteByName(mfg, "reader").Replicating = false if err := env.Kube.Controller.Update(context.Background(), mfg); err != nil { t.Fatal(err) } diff --git a/internal/playground/scenarios/s40_reader_data_loss.go b/internal/playground/scenarios/s40_reader_data_loss.go index d31fa87..9431a42 100644 --- a/internal/playground/scenarios/s40_reader_data_loss.go +++ b/internal/playground/scenarios/s40_reader_data_loss.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" @@ -88,11 +87,11 @@ func s40Precheck(state *s40RunState) func(context.Context, *runner.Env) error { } state.donorHost = playgroundInternalSiteHost(env.FG, state.active, env.Namespace) - readerStatus := s40StatusSite(mfg, state.reader) + readerStatus := statusSiteByName(mfg, state.reader) if readerStatus == nil { return fmt.Errorf("reader %q missing from status", state.reader) } - if err := s40AssertReaderStatus(mfg, readerStatus, state.donorHost); err != nil { + if err := assertReaderServingStatus(mfg, readerStatus, state.donorHost); err != nil { return fmt.Errorf("reader status precheck: %w", err) } pod, err := env.Kube.GetSiteMysqlPod(ctx, env.Namespace, env.FG, state.reader) @@ -131,24 +130,10 @@ func s40SeedAndReplicateMarker(state *s40RunState) runner.Step { Name: "write a marker on the active primary and confirm it reaches the reader", Do: func(ctx context.Context, env *runner.Env) error { state.marker = fmt.Sprintf("reader-loss-%d", time.Now().UnixNano()) - active, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, state.active, env.Creds) - if err != nil { - return fmt.Errorf("open active MySQL %s: %w", state.active, err) - } - defer active.Close() - for _, stmt := range []struct { - query string - args []any - }{ - {query: "CREATE DATABASE IF NOT EXISTS bloodraven_reader_e2e"}, - {query: "CREATE TABLE IF NOT EXISTS " + s40MarkerTable + " (marker VARCHAR(128) PRIMARY KEY)"}, - {query: "INSERT INTO " + s40MarkerTable + " (marker) VALUES (?)", args: []any{state.marker}}, - } { - if _, err := active.Exec(ctx, stmt.query, stmt.args...); err != nil { - return fmt.Errorf("write marker %q: %w", state.marker, err) - } + if err := seedMarkerRow(ctx, env, state.active, s40MarkerTable, state.marker); err != nil { + return err } - return s40WaitForMarker(ctx, env, state.reader, state.marker, 60*time.Second) + return waitForMarkerOnSite(ctx, env, state.reader, s40MarkerTable, state.marker, 60*time.Second) }, } } @@ -222,11 +207,11 @@ func s40ReplaceReaderDataAndObserve(state *s40RunState) runner.Step { if mfg.Status.ActiveSite != state.active { return false, fmt.Sprintf("activeSite=%q", mfg.Status.ActiveSite), fmt.Errorf("active site changed during reader loss: %q -> %q", state.active, mfg.Status.ActiveSite) } - status := s40StatusSite(mfg, state.reader) + status := statusSiteByName(mfg, state.reader) if status == nil { return false, "reader status missing", nil } - err := s40AssertReaderStatus(mfg, status, state.donorHost) + err := assertReaderServingStatus(mfg, status, state.donorHost) return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s lag=%v", status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SecondsBehindSource), nil }) cancelWait() @@ -293,8 +278,8 @@ func s40ObserveOnce(ctx context.Context, env *runner.Env, state *s40RunState, ob if mfg.Status.ActiveSite != state.active { observation.fail(fmt.Errorf("active site changed while reader was unavailable/recovering: %q -> %q", state.active, mfg.Status.ActiveSite)) } - if status := s40StatusSite(mfg, state.reader); status != nil { - readerServingHealthy = s40AssertReaderStatus(mfg, status, state.donorHost) == nil + if status := statusSiteByName(mfg, state.reader); status != nil { + readerServingHealthy = assertReaderServingStatus(mfg, status, state.donorHost) == nil } } @@ -387,29 +372,6 @@ func (o *s40ContinuousObservation) result() error { return nil } -func s40AssertReaderStatus(mfg *v1alpha1.MysqlFailoverGroup, status *v1alpha1.SiteStatus, expectedHost string) error { - if status.State != "read-only" { - return fmt.Errorf("state=%q, want read-only", status.State) - } - if !status.Replicating { - return fmt.Errorf("replicating=false") - } - if status.SourceConvergenceState != v1alpha1.SourceConvergenceConverged { - return fmt.Errorf("sourceConvergenceState=%q, want Converged", status.SourceConvergenceState) - } - if canonicalMySQLHost(status.SourceHost) != canonicalMySQLHost(expectedHost) { - return fmt.Errorf("sourceHost=%q, want %q", status.SourceHost, expectedHost) - } - if status.SecondsBehindSource == nil { - return fmt.Errorf("secondsBehindSource is unknown") - } - maxLag := mfg.Spec.EffectiveReadOnlyMaxLagSeconds() - if *status.SecondsBehindSource > maxLag { - return fmt.Errorf("secondsBehindSource=%d exceeds reader threshold %d", *status.SecondsBehindSource, maxLag) - } - return nil -} - func s40WaitForLiveReader(ctx context.Context, env *runner.Env, state *s40RunState, timeout time.Duration) error { deadline := time.Now().Add(timeout) var last string @@ -435,30 +397,6 @@ func s40WaitForLiveReader(ctx context.Context, env *runner.Env, state *s40RunSta return fmt.Errorf("reader live replication did not converge within %s (last: %s)", timeout, last) } -func s40WaitForMarker(ctx context.Context, env *runner.Env, site, marker string, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - var last error - for time.Now().Before(deadline) { - client, err := pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, site, env.Creds) - if err == nil { - count, queryErr := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+s40MarkerTable+" WHERE marker=?", marker) - _ = client.Close() - if queryErr == nil && count == 1 { - return nil - } - last = queryErr - } else { - last = err - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Second): - } - } - return fmt.Errorf("marker %q did not replicate to %s within %s: %v", marker, site, timeout, last) -} - func s40AssertServiceTopology(ctx context.Context, env *runner.Env, reader, expectedPod string, requireClientEndpoint bool) error { clientName := pgkube.MysqlDeploymentName(env.FG, reader) clientService, err := env.Kube.Kubernetes.CoreV1().Services(env.Namespace).Get(ctx, clientName, metav1.GetOptions{}) @@ -521,25 +459,6 @@ func s40WaitForServiceTopology(ctx context.Context, env *runner.Env, reader, exp return fmt.Errorf("reader Service topology did not converge within %s: %w", timeout, last) } -func s40StatusSite(mfg *v1alpha1.MysqlFailoverGroup, name string) *v1alpha1.SiteStatus { - for i := range mfg.Status.Sites { - if mfg.Status.Sites[i].Name == name { - return &mfg.Status.Sites[i] - } - } - return nil -} - -func playgroundInternalSiteHost(group, site, namespace string) string { - return fmt.Sprintf("mysql-%s-%s-internal.%s.svc.cluster.local", group, site, namespace) -} - -func canonicalMySQLHost(host string) string { - host = strings.ToLower(strings.TrimSpace(host)) - host = strings.TrimSuffix(host, ":3306") - return strings.TrimSuffix(host, ".") -} - func s40PodReady(pod *corev1.Pod) bool { for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodReady { diff --git a/internal/playground/scenarios/s41_reader_availability_during_failover.go b/internal/playground/scenarios/s41_reader_availability_during_failover.go new file mode 100644 index 0000000..e6afb57 --- /dev/null +++ b/internal/playground/scenarios/s41_reader_availability_during_failover.go @@ -0,0 +1,323 @@ +package scenarios + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgmysql "github.com/shipstream/bloodraven/internal/playground/mysql" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +const s41MarkerTable = "bloodraven_reader_e2e.failover_marker" + +func init() { + runner.Register(scenario41ReaderAvailabilityDuringFailover()) +} + +type s41RunState struct { + topo readerTopology + oldHost string + newHost string + marker string +} + +// s41Observation is the continuous reader-availability record taken +// through the unplanned-failover window. Staleness is allowed; +// availability is required — a single hard read failure (after one +// reconnect attempt) fails the scenario. The observer also tracks the +// reader's reported replication source so the scenario can assert the +// reader only ever moved old-primary → new-primary and never entered a +// blocked state. +type s41Observation struct { + mu sync.Mutex + err error + reads int + sourceHosts []string +} + +// scenario41ReaderAvailabilityDuringFailover is chaos proposal R3 from +// issue #115: during an unplanned failover the reader keeps serving +// (stale) reads, and after promotion the source-convergence pass +// repoints it directly at the new primary. +func scenario41ReaderAvailabilityDuringFailover() runner.Scenario { + state := &s41RunState{} + return runner.Scenario{ + ID: "41-reader-availability-during-failover", + Title: "Reader keeps serving through unplanned failover, then repoints to the new primary", + Hypothesis: "Hard-killing the active primary (sustained scale-to-0) leaves the reader answering SELECTs " + + "throughout the failover window; after promotion the operator repoints the reader directly at the " + + "new primary with no blocked or chained intermediate state.", + Risk: "medium", + DocLink: "playground/chaos-scenarios.md#41-reader-availability-during-unplanned-failover", + Timeout: 8 * time.Minute, + Precheck: s41Precheck(state), + Steps: []runner.Step{ + s41SeedMarker(state), + s41KillPrimaryAndObserveReader(state), + s41VerifyReaderEndpointRestored(state), + }, + Cleanup: s08AutoRecloneCleanup, + } +} + +func s41Precheck(state *s41RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + *state = s41RunState{} + topo, err := resolveReaderTopology(ctx, env) + if err != nil { + return err + } + state.topo = topo + state.oldHost = topo.activeHost + state.newHost = playgroundInternalSiteHost(env.FG, topo.standby, env.Namespace) + return nil + } +} + +func s41SeedMarker(state *s41RunState) runner.Step { + return runner.Step{ + Phase: runner.PhasePrecheck, + Name: "write a marker on the active primary and confirm it reaches the reader", + Do: func(ctx context.Context, env *runner.Env) error { + state.marker = fmt.Sprintf("reader-failover-%d", time.Now().UnixNano()) + if err := seedMarkerRow(ctx, env, state.topo.active, s41MarkerTable, state.marker); err != nil { + return err + } + return waitForMarkerOnSite(ctx, env, state.topo.reader, s41MarkerTable, state.marker, 60*time.Second) + }, + } +} + +func s41KillPrimaryAndObserveReader(state *s41RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseInject, + Name: "scale active primary to 0; reader must answer SELECTs continuously and repoint to the new primary", + Do: func(ctx context.Context, env *runner.Env) error { + observeCtx, stopObserve := context.WithCancel(ctx) + observation := &s41Observation{} + observeDone := make(chan struct{}) + go func() { + defer close(observeDone) + s41ObserveContinuously(observeCtx, env, state, observation) + }() + stopAndCheck := func() error { + stopObserve() + <-observeDone + return observation.result(state) + } + + env.Capture.Note(fmt.Sprintf("killing active primary %s (sustained scale-to-0); expecting failover to %s", state.topo.active, state.topo.standby)) + if err := env.Chaos.ScaleSiteToZero(ctx, state.topo.active); err != nil { + _ = stopAndCheck() + return err + } + + flipCtx, cancelFlip := context.WithTimeout(ctx, 2*time.Minute) + _, err := env.Wait.UntilCR(flipCtx, env.Namespace, + fmt.Sprintf("activeSite flips %s -> %s", state.topo.active, state.topo.standby), + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + msg := fmt.Sprintf("activeSite=%q lastFailoverTarget=%q", mfg.Status.ActiveSite, mfg.Status.LastFailoverTarget) + if mfg.Status.ActiveSite == "" || mfg.Status.ActiveSite == state.topo.active { + return false, msg, nil + } + if mfg.Status.ActiveSite != state.topo.standby { + return false, msg, fmt.Errorf("failover landed on %q, want promotable standby %q", mfg.Status.ActiveSite, state.topo.standby) + } + return true, msg, nil + }) + cancelFlip() + if err != nil { + _ = stopAndCheck() + return err + } + + repointCtx, cancelRepoint := context.WithTimeout(ctx, 3*time.Minute) + _, err = env.Wait.UntilCR(repointCtx, env.Namespace, + "reader converges directly onto the new primary", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + if mfg.Status.ActiveSite != state.topo.standby { + return false, "", fmt.Errorf("active site changed again during reader repoint: %q", mfg.Status.ActiveSite) + } + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return false, "reader status missing", nil + } + err := assertReaderServingStatus(mfg, status, state.newHost) + return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s/%s lag=%v", + status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SourceConvergenceReason, status.SecondsBehindSource), nil + }) + cancelRepoint() + if err != nil { + _ = stopAndCheck() + return err + } + + // Live write-through: a marker on the new primary must reach the + // reader via the repointed direct channel. + postMarker := state.marker + "-post" + if err := seedMarkerRow(ctx, env, state.topo.standby, s41MarkerTable, postMarker); err != nil { + _ = stopAndCheck() + return err + } + if err := waitForMarkerOnSite(ctx, env, state.topo.reader, s41MarkerTable, postMarker, 90*time.Second); err != nil { + _ = stopAndCheck() + return err + } + + if err := stopAndCheck(); err != nil { + return err + } + observation.mu.Lock() + reads, hosts := observation.reads, observation.sourceHosts + observation.mu.Unlock() + env.Capture.Note(fmt.Sprintf("reader answered %d reads through the failover window; source history=%v", reads, hosts)) + return nil + }, + } +} + +// s41VerifyReaderEndpointRestored asserts the reader's client Service +// publishes exactly the reader pod again once it is serving from the +// new primary. +func s41VerifyReaderEndpointRestored(state *s41RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "reader client Service publishes exactly the reader pod again", + Do: func(ctx context.Context, env *runner.Env) error { + return waitReaderClientEndpoint(ctx, env, state.topo.reader, 90*time.Second) + }, + } +} + +func s41ObserveContinuously(ctx context.Context, env *runner.Env, state *s41RunState, observation *s41Observation) { + var client *pgmysql.SiteClient + defer func() { + if client != nil { + _ = client.Close() + } + }() + readMarker := func() error { + var err error + if client == nil { + client, err = pgmysql.Open(ctx, env.Kube, env.Namespace, env.FG, state.topo.reader, env.Creds) + if err != nil { + return fmt.Errorf("open reader connection: %w", err) + } + } + count, err := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+s41MarkerTable+" WHERE marker=?", state.marker) + if err != nil { + return err + } + if count != 1 { + return fmt.Errorf("marker row missing (count=%d)", count) + } + return nil + } + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if ctx.Err() != nil { + return + } + if err := readMarker(); err != nil && ctx.Err() == nil { + // One reconnect attempt: an idle port-forward tunnel can drop + // without the reader itself being unavailable. Two consecutive + // failures count as a real availability gap. + if client != nil { + _ = client.Close() + client = nil + } + if retryErr := readMarker(); retryErr != nil && ctx.Err() == nil { + observation.fail(fmt.Errorf("reader stopped answering reads during failover: %v (after reconnect: %v)", err, retryErr)) + } + } else if err == nil { + observation.mu.Lock() + observation.reads++ + observation.mu.Unlock() + } + + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + if ctx.Err() == nil && !errors.Is(err, ctx.Err()) { + observation.fail(fmt.Errorf("continuous observer read MFG: %w", err)) + } + } else if status := statusSiteByName(mfg, state.topo.reader); status != nil { + if status.SourceConvergenceState == v1alpha1.SourceConvergenceBlocked { + observation.fail(fmt.Errorf("reader entered SourceConvergenceState=Blocked (reason=%s) during failover", status.SourceConvergenceReason)) + } + if status.RecoveryState == "RecoveryBlocked" { + observation.fail(fmt.Errorf("reader entered RecoveryBlocked during failover")) + } + observation.recordSourceHost(status.SourceHost) + } + + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (o *s41Observation) fail(err error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.err == nil { + o.err = err + } +} + +func (o *s41Observation) recordSourceHost(host string) { + canonical := canonicalMySQLHost(host) + if canonical == "" { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if n := len(o.sourceHosts); n == 0 || o.sourceHosts[n-1] != canonical { + o.sourceHosts = append(o.sourceHosts, canonical) + } +} + +// result validates the whole observation window: no availability gap, +// enough samples to be meaningful, and a source-host history that only +// ever moves old-primary → new-primary. Any third host, or a return to +// the demoted primary after the repoint, means the reader applied +// events through a chained or stale channel. +func (o *s41Observation) result(state *s41RunState) error { + o.mu.Lock() + defer o.mu.Unlock() + if o.err != nil { + return o.err + } + if o.reads < 20 { + return fmt.Errorf("availability observer completed only %d reads; too few to cover the failover window", o.reads) + } + oldHost := canonicalMySQLHost(state.oldHost) + newHost := canonicalMySQLHost(state.newHost) + sawNew := false + for _, host := range o.sourceHosts { + switch host { + case oldHost: + if sawNew { + return fmt.Errorf("reader source flipped back to demoted primary after repoint: history=%v", o.sourceHosts) + } + case newHost: + sawNew = true + default: + return fmt.Errorf("reader reported unexpected replication source %q (history=%v)", host, o.sourceHosts) + } + } + if !sawNew { + return fmt.Errorf("reader never reported the new primary as its source: history=%v", o.sourceHosts) + } + if n := len(o.sourceHosts); o.sourceHosts[n-1] != newHost { + return fmt.Errorf("reader source history does not end on the new primary: %v", o.sourceHosts) + } + return nil +} diff --git a/internal/playground/scenarios/s42_reader_stall_isolation.go b/internal/playground/scenarios/s42_reader_stall_isolation.go new file mode 100644 index 0000000..49b730c --- /dev/null +++ b/internal/playground/scenarios/s42_reader_stall_isolation.go @@ -0,0 +1,301 @@ +package scenarios + +import ( + "context" + "fmt" + "time" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + pgmetrics "github.com/shipstream/bloodraven/internal/playground/metrics" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +const ( + s42DBName = "chaos_s42" + s42TableName = s42DBName + ".stall_rows" + // s42SourceDelaySeconds must exceed the soak duration so applied lag + // keeps growing for the whole observation window. SOURCE_DELAY (not + // STOP REPLICA SQL_THREAD) is the one lag injection the operator + // will not heal: both replication threads keep running on the + // correct source, so the source-convergence invariant sees a + // converged-but-lagging reader instead of a stopped one. + s42SourceDelaySeconds = 600 +) + +func init() { + runner.Register(scenario42ReaderStallIsolation()) +} + +type s42RunState struct { + topo readerTopology + // lastFailoverBefore is the pre-injection status.lastFailover + // timestamp ("" when unset). Any change during the soak means the + // stalled reader consumed a failover / anti-flap cooldown. + lastFailoverBefore string + sawEndpointEmpty bool + maxObservedLag int64 +} + +// scenario42ReaderStallIsolation is chaos proposal R4 from issue #115: +// a wedged OLAP reader is invisible to the failover machinery. Lag +// grows without bound with zero group-level effect; the only reaction +// is endpoint shedding plus alertable per-site status and metrics. +func scenario42ReaderStallIsolation() runner.Scenario { + state := &s42RunState{} + return runner.Scenario{ + ID: "42-reader-stall-no-group-degradation", + Title: "Stalled reader sheds its endpoint without degrading the group", + Hypothesis: "SOURCE_DELAY on the reader grows replication lag past readOnlyMaxLagSeconds while both threads " + + "keep running. Group Ready stays True for a soak of 3x maxLagSeconds with no failover and no cooldown " + + "consumed; the reader leaves its client Service endpoints and status/metrics report the stall.", + Risk: "low", + DocLink: "playground/chaos-scenarios.md#42-reader-stall-does-not-degrade-the-group", + Timeout: 10 * time.Minute, + Precheck: s42Precheck(state), + Steps: []runner.Step{ + s42InjectSourceDelay(state), + s42SoakGroupUnaffected(state), + s42VerifyStallObservability(state), + s42ClearDelayAndRecover(state), + }, + Cleanup: s42Cleanup(state), + } +} + +func s42Precheck(state *s42RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + *state = s42RunState{} + topo, err := resolveReaderTopology(ctx, env) + if err != nil { + return err + } + state.topo = topo + return nil + } +} + +func s42SetSourceDelay(ctx context.Context, env *runner.Env, reader string, seconds int) error { + client, err := env.MySQL(reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + // One multi-statement batch keeps the stopped-replica window tiny so + // the operator's convergence pass has no time to observe (and + // restart) a stopped thread mid-injection. + stmt := fmt.Sprintf("STOP REPLICA; CHANGE REPLICATION SOURCE TO SOURCE_DELAY = %d; START REPLICA", seconds) + if _, err := client.Exec(ctx, stmt); err != nil { + return fmt.Errorf("set SOURCE_DELAY=%d on %s: %w", seconds, reader, err) + } + return nil +} + +func s42InjectSourceDelay(state *s42RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseInject, + Name: "apply SOURCE_DELAY on the reader only", + Do: func(ctx context.Context, env *runner.Env) error { + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + if mfg.Status.LastFailover != nil { + state.lastFailoverBefore = mfg.Status.LastFailover.Time.UTC().Format(time.RFC3339) + } + primary, err := env.MySQL(state.topo.active) + if err != nil { + return fmt.Errorf("open primary mysql: %w", err) + } + if _, err := primary.Exec(ctx, "CREATE DATABASE IF NOT EXISTS "+s42DBName+ + "; CREATE TABLE IF NOT EXISTS "+s42TableName+" (id BIGINT PRIMARY KEY, written_at TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3))"); err != nil { + return fmt.Errorf("create stall table: %w", err) + } + if err := s42SetSourceDelay(ctx, env, state.topo.reader, s42SourceDelaySeconds); err != nil { + return err + } + env.Capture.Note(fmt.Sprintf("SOURCE_DELAY=%ds applied on reader %s; lastFailoverBefore=%q", + s42SourceDelaySeconds, state.topo.reader, state.lastFailoverBefore)) + return nil + }, + } +} + +// s42SoakGroupUnaffected writes a row per second to the primary while +// continuously asserting the group is untouched by the reader stall: +// Ready stays True, the active site never changes, no failover happens +// and no anti-flap cooldown is consumed, and the reader never enters a +// blocked convergence or recovery state. The reader must leave its +// client Service endpoints once its lag exceeds readOnlyMaxLagSeconds. +func s42SoakGroupUnaffected(state *s42RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseObserve, + Name: "soak 3x maxLagSeconds: group Ready, no failover, reader sheds endpoint as lag grows", + Do: func(ctx context.Context, env *runner.Env) error { + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + soak := time.Duration(3*mfg.Spec.EffectiveMaxLagSeconds()) * time.Second + readerMaxLag := mfg.Spec.EffectiveReadOnlyMaxLagSeconds() + env.Capture.Note(fmt.Sprintf("soaking %s (3x maxLagSeconds), reader lag threshold %ds", soak, readerMaxLag)) + + primary, err := env.MySQL(state.topo.active) + if err != nil { + return fmt.Errorf("open primary mysql: %w", err) + } + deadline := time.Now().Add(soak) + for i := 0; time.Now().Before(deadline); i++ { + if _, err := primary.Exec(ctx, "INSERT INTO "+s42TableName+" (id) VALUES (?)", time.Now().UnixNano()); err != nil { + return fmt.Errorf("soak write %d on primary: %w", i, err) + } + + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return fmt.Errorf("soak read MFG: %w", err) + } + if ready := pgkube.ReadyCondition(mfg); ready != "True" { + return fmt.Errorf("group Ready flipped to %q while only the reader was stalled", ready) + } + if mfg.Status.ActiveSite != state.topo.active { + return fmt.Errorf("active site changed during reader stall: %q -> %q", state.topo.active, mfg.Status.ActiveSite) + } + lastFailover := "" + if mfg.Status.LastFailover != nil { + lastFailover = mfg.Status.LastFailover.Time.UTC().Format(time.RFC3339) + } + if lastFailover != state.lastFailoverBefore { + return fmt.Errorf("failover recorded during reader stall: lastFailover %q -> %q", state.lastFailoverBefore, lastFailover) + } + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return fmt.Errorf("reader %q missing from status during soak", state.topo.reader) + } + if status.State != "read-only" { + return fmt.Errorf("reader state=%q during stall, want read-only", status.State) + } + if status.SourceConvergenceState == v1alpha1.SourceConvergenceBlocked { + return fmt.Errorf("reader entered SourceConvergenceState=Blocked (reason=%s) during a pure lag stall", status.SourceConvergenceReason) + } + if status.RecoveryState == "RecoveryBlocked" { + return fmt.Errorf("reader entered RecoveryBlocked during a pure lag stall") + } + if status.SecondsBehindSource != nil && *status.SecondsBehindSource > state.maxObservedLag { + state.maxObservedLag = *status.SecondsBehindSource + } + standby := statusSiteByName(mfg, state.topo.standby) + if standby == nil || !standby.Replicating { + return fmt.Errorf("promotable standby %s stopped replicating during reader stall", state.topo.standby) + } + + endpoints, err := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.topo.reader)) + if err != nil { + return fmt.Errorf("soak read reader client EndpointSlices: %w", err) + } + if len(endpoints.ReadyPodNames("mysql")) == 0 { + state.sawEndpointEmpty = true + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + + if state.maxObservedLag <= readerMaxLag { + return fmt.Errorf("reader lag never exceeded readOnlyMaxLagSeconds=%d during soak (max observed %d)", readerMaxLag, state.maxObservedLag) + } + if !state.sawEndpointEmpty { + return fmt.Errorf("reader client Service never shed its endpoint although lag reached %ds (threshold %ds)", state.maxObservedLag, readerMaxLag) + } + env.Capture.Note(fmt.Sprintf("soak complete: max observed reader lag %ds, endpoint shed observed, group untouched", state.maxObservedLag)) + return nil + }, + } +} + +// s42VerifyStallObservability asserts the stall is alertable: the lag +// gauge is over threshold and the source-state metric still reports the +// stalled reader as converged (it is on the right source — just slow). +func s42VerifyStallObservability(state *s42RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "lag gauge exceeds reader threshold while source-state stays converged", + Do: func(ctx context.Context, env *runner.Env) error { + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + readerMaxLag := float64(mfg.Spec.EffectiveReadOnlyMaxLagSeconds()) + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + return env.Wait.UntilMetric(waitCtx, env.Metrics, + fmt.Sprintf("replication_lag_seconds{site=%q} > %g and replication_source_state{site=%q,state=\"converged\"} == 1", + state.topo.reader, readerMaxLag, state.topo.reader), + func(snap *pgmetrics.Snapshot) (bool, string) { + lag, lagOK := snap.Gauge("bloodraven_replication_lag_seconds", map[string]string{"site": state.topo.reader}) + converged, stateOK := snap.Gauge("bloodraven_replication_source_state", map[string]string{"site": state.topo.reader, "state": "converged"}) + msg := fmt.Sprintf("lag=%g(ok=%v) converged=%g(ok=%v)", lag, lagOK, converged, stateOK) + return lagOK && lag > readerMaxLag && stateOK && converged == 1, msg + }, + ) + }, + } +} + +func s42ClearDelayAndRecover(state *s42RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseSettle, + Name: "clear SOURCE_DELAY; reader catches up and rejoins its client Service", + Do: func(ctx context.Context, env *runner.Env) error { + if err := s42SetSourceDelay(ctx, env, state.topo.reader, 0); err != nil { + return err + } + waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + _, err := env.Wait.UntilCR(waitCtx, env.Namespace, + "reader lag back under threshold with serving status", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return false, "reader status missing", nil + } + err := assertReaderServingStatus(mfg, status, state.topo.activeHost) + return err == nil, fmt.Sprintf("state=%s replicating=%v convergence=%s lag=%v", + status.State, status.Replicating, status.SourceConvergenceState, status.SecondsBehindSource), nil + }) + if err != nil { + return err + } + return waitReaderClientEndpoint(ctx, env, state.topo.reader, 90*time.Second) + }, + } +} + +// s42Cleanup is defensive: on any failure path it clears the delay so a +// wedged reader cannot leak into the next scenario, then drops the soak +// database through the primary (the drop replicates to every follower). +func s42Cleanup(state *s42RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + if state.topo.reader != "" { + if err := s42SetSourceDelay(ctx, env, state.topo.reader, 0); err != nil { + return fmt.Errorf("cleanup: clear SOURCE_DELAY: %w", err) + } + } + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return fmt.Errorf("cleanup: get MFG: %w", err) + } + if mfg.Status.ActiveSite == "" { + return fmt.Errorf("cleanup: no active site to drop %s through", s42DBName) + } + primary, err := env.MySQL(mfg.Status.ActiveSite) + if err != nil { + return fmt.Errorf("cleanup: open primary mysql: %w", err) + } + if _, err := primary.Exec(ctx, "DROP DATABASE IF EXISTS "+s42DBName); err != nil { + return fmt.Errorf("cleanup: drop %s: %w", s42DBName, err) + } + return nil + } +} diff --git a/internal/playground/scenarios/s43_writable_reader_fence.go b/internal/playground/scenarios/s43_writable_reader_fence.go new file mode 100644 index 0000000..913601a --- /dev/null +++ b/internal/playground/scenarios/s43_writable_reader_fence.go @@ -0,0 +1,489 @@ +package scenarios + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + pglogs "github.com/shipstream/bloodraven/internal/playground/logs" + pgmetrics "github.com/shipstream/bloodraven/internal/playground/metrics" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +const s43DBName = "chaos_s43" + +func init() { + runner.Register(scenario43WritableReaderFence()) +} + +type s43RunState struct { + topo readerTopology + readerUUID string + rejectedBefore float64 + errantSet string + recovered bool +} + +// scenario43WritableReaderFence is chaos proposal R5 from issue #115: +// role semantics hold under the worst input. A reader that somehow +// becomes writable is fenced like a dr-only loser, its errant GTID +// blocks source convergence at the GTID gate, and it is never a +// promotion target — not even by explicit admin request. +func scenario43WritableReaderFence() runner.Scenario { + state := &s43RunState{} + return runner.Scenario{ + ID: "43-writable-reader-fence", + Title: "Anomalously writable reader is fenced, blocked on divergence, and never promoted", + Hypothesis: "Turning off super_read_only on the reader and writing an errant row triggers the un-debounced " + + "fence back to super_read_only=ON; a planned failover targeting the reader is rejected with the role " + + "error; and the errant GTID trips the convergence GTID gate into Blocked/GTIDDiverged instead of a " + + "silent repoint.", + Risk: "medium", + DocLink: "playground/chaos-scenarios.md#43-writable-reader-fence", + Timeout: 10 * time.Minute, + Precheck: s43Precheck(state), + Steps: []runner.Step{ + s43InjectWritableReaderWithErrantRow(state), + s43ObserveFence(state), + s43VerifyPlannedFailoverRejected(state), + s43VerifyErrantGtidBlocksConvergence(state), + s43ReconcileAndRecover(state), + }, + Cleanup: s43Cleanup(state), + } +} + +func s43Precheck(state *s43RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + *state = s43RunState{} + topo, err := resolveReaderTopology(ctx, env) + if err != nil { + return err + } + state.topo = topo + return nil + } +} + +func s43InjectWritableReaderWithErrantRow(state *s43RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseInject, + Name: "turn off super_read_only on the reader and write one errant row", + Do: func(ctx context.Context, env *runner.Env) error { + // Open the operator tailer before injecting so the fence line + // cannot scroll past the SinceTime window. + if _, err := env.Logs("operator"); err != nil { + return fmt.Errorf("open operator tailer: %w", err) + } + before, err := metricCounter(ctx, env, "bloodraven_planned_failovers_total", map[string]string{ + "target_site": state.topo.reader, + "result": "rejected", + }) + if err != nil { + return err + } + state.rejectedBefore = before + + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + state.readerUUID, err = reader.ScalarString(ctx, "SELECT @@server_uuid") + if err != nil { + return fmt.Errorf("read reader server_uuid: %w", err) + } + // One multi-statement batch: the operator fences a writable + // non-promotable site without debounce, so the errant write has + // to land before the next poll cycle can flip the flag back. + stmt := "SET GLOBAL super_read_only = OFF; SET GLOBAL read_only = OFF" + + "; CREATE DATABASE " + s43DBName + + "; CREATE TABLE " + s43DBName + ".errant (id INT PRIMARY KEY)" + + "; INSERT INTO " + s43DBName + ".errant (id) VALUES (1)" + if _, err := reader.Exec(ctx, stmt); err != nil { + return fmt.Errorf("errant write on reader: %w", err) + } + env.Capture.Note(fmt.Sprintf("reader %s made writable and holds errant transactions from %s", state.topo.reader, state.readerUUID)) + return nil + }, + } +} + +func s43ObserveFence(state *s43RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseObserve, + Name: "operator fences the reader back to super_read_only without group impact", + Do: func(ctx context.Context, env *runner.Env) error { + tail, err := env.Logs("operator") + if err != nil { + return err + } + logCtx, cancelLog := context.WithTimeout(ctx, 60*time.Second) + _, err = env.Wait.UntilLog(logCtx, tail, env.StartTime, + "writable non-promotable reader is fenced", + pglogs.Structured("fenced writable non-promotable site", map[string]string{"site": state.topo.reader})) + cancelLog() + if err != nil { + return err + } + + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + deadline := time.Now().Add(30 * time.Second) + for { + on, err := reader.SuperReadOnly(ctx) + if err == nil && on { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("reader super_read_only not restored within 30s (on=%v err=%v)", on, err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + if ready := pgkube.ReadyCondition(mfg); ready != "True" { + return fmt.Errorf("group Ready=%q after reader fence, want True", ready) + } + if mfg.Status.ActiveSite != state.topo.active { + return fmt.Errorf("active site changed after reader fence: %q -> %q", state.topo.active, mfg.Status.ActiveSite) + } + return nil + }, + } +} + +func s43VerifyPlannedFailoverRejected(state *s43RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "planned failover targeting the reader is rejected with the role error", + Do: func(ctx context.Context, env *runner.Env) error { + if err := env.Chaos.AnnotatePlannedFailover(ctx, state.topo.reader); err != nil { + return err + } + waitCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + _, err := env.Wait.UntilCR(waitCtx, env.Namespace, + "plannedFailover Failed with the only-primary-candidate role error", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + pf := mfg.Status.PlannedFailover + if pf == nil { + return false, "no plannedFailover status yet", nil + } + // Ignore terminal blocks left over from earlier scenarios + // (metav1.Time truncates to seconds; 2s of slack). + staleCutoff := env.StartTime.Add(-2 * time.Second) + if pf.StartTime == nil || pf.StartTime.Time.Before(staleCutoff) { + return false, fmt.Sprintf("ignoring stale plannedFailover (startTime=%v)", pf.StartTime), nil + } + msg := fmt.Sprintf("phase=%q target=%q reason=%q message=%q", pf.Phase, pf.Target, pf.Reason, pf.Message) + if pf.Phase != v1alpha1.PlannedFailoverPhaseFailed { + if pf.Phase == v1alpha1.PlannedFailoverPhaseSucceeded { + return false, msg, fmt.Errorf("planned failover to reader %q succeeded; role gate is broken", state.topo.reader) + } + return false, msg, nil + } + if pf.Target != state.topo.reader { + return false, msg, fmt.Errorf("rejected plannedFailover has target=%q, want %q", pf.Target, state.topo.reader) + } + if !strings.Contains(pf.Message, "only primary-candidate sites may be promoted") { + return false, msg, fmt.Errorf("rejection message %q does not carry the role error", pf.Message) + } + return true, msg, nil + }) + if err != nil { + return err + } + + metricCtx, cancelMetric := context.WithTimeout(ctx, 30*time.Second) + defer cancelMetric() + return env.Wait.UntilMetric(metricCtx, env.Metrics, + fmt.Sprintf(`planned_failovers_total{target_site=%q,result="rejected"} increments from %g`, state.topo.reader, state.rejectedBefore), + func(snap *pgmetrics.Snapshot) (bool, string) { + v, _ := snap.Counter("bloodraven_planned_failovers_total", map[string]string{ + "target_site": state.topo.reader, + "result": "rejected", + }) + return v > state.rejectedBefore, fmt.Sprintf("counter=%g before=%g", v, state.rejectedBefore) + }, + ) + }, + } +} + +// s43VerifyErrantGtidBlocksConvergence stops the reader's replication so +// the source-convergence invariant has to act, and asserts the GTID +// containment gate refuses to restart a diverged follower: state goes +// Blocked/GTIDDiverged with the documented log line, the blocked metric +// flips, and the reader is shed from its client Service. +func s43VerifyErrantGtidBlocksConvergence(state *s43RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "errant GTID trips the convergence gate: Blocked/GTIDDiverged, never a silent restart", + Do: func(ctx context.Context, env *runner.Env) error { + active, err := env.MySQL(state.topo.active) + if err != nil { + return fmt.Errorf("open active mysql: %w", err) + } + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + readerGtid, err := reader.GtidExecuted(ctx) + if err != nil { + return err + } + activeGtid, err := active.GtidExecuted(ctx) + if err != nil { + return err + } + state.errantSet, err = active.ScalarString(ctx, "SELECT GTID_SUBTRACT(?, ?)", readerGtid, activeGtid) + if err != nil { + return fmt.Errorf("compute errant GTID set: %w", err) + } + state.errantSet = strings.ReplaceAll(strings.TrimSpace(state.errantSet), "\n", "") + if state.errantSet == "" { + return fmt.Errorf("reader holds no errant transactions after the errant write; injection did not take") + } + env.Capture.Note(fmt.Sprintf("reader errant GTID set: %s", state.errantSet)) + + tail, err := env.Logs("operator") + if err != nil { + return err + } + if _, err := reader.Exec(ctx, "STOP REPLICA"); err != nil { + return fmt.Errorf("stop replica on reader: %w", err) + } + + logCtx, cancelLog := context.WithTimeout(ctx, 90*time.Second) + _, err = env.Wait.UntilLog(logCtx, tail, env.StartTime, + "convergence blocked on GTID containment", + pglogs.Structured("replication source convergence blocked", map[string]string{"site": state.topo.reader})) + cancelLog() + if err != nil { + return err + } + + waitCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + mfg, err := env.Wait.UntilCR(waitCtx, env.Namespace, + "reader sourceConvergenceState=Blocked reason=GTIDDiverged", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return false, "reader status missing", nil + } + msg := fmt.Sprintf("convergence=%s/%s replicating=%v", status.SourceConvergenceState, status.SourceConvergenceReason, status.Replicating) + done := status.SourceConvergenceState == v1alpha1.SourceConvergenceBlocked && status.SourceConvergenceReason == "GTIDDiverged" + return done, msg, nil + }) + cancel() + if err != nil { + return err + } + if ready := pgkube.ReadyCondition(mfg); ready != "True" { + return fmt.Errorf("group Ready=%q while only the reader is blocked, want True", ready) + } + if mfg.Status.ActiveSite != state.topo.active { + return fmt.Errorf("active site changed while reader was blocked: %q -> %q", state.topo.active, mfg.Status.ActiveSite) + } + + metricCtx, cancelMetric := context.WithTimeout(ctx, 30*time.Second) + defer cancelMetric() + if err := env.Wait.UntilMetric(metricCtx, env.Metrics, + fmt.Sprintf(`replication_source_state{site=%q,state="blocked"} == 1`, state.topo.reader), + func(snap *pgmetrics.Snapshot) (bool, string) { + v, ok := snap.Gauge("bloodraven_replication_source_state", map[string]string{"site": state.topo.reader, "state": "blocked"}) + return ok && v == 1, fmt.Sprintf("blocked=%g(ok=%v)", v, ok) + }, + ); err != nil { + return err + } + + deadline := time.Now().Add(60 * time.Second) + var last string + for time.Now().Before(deadline) { + endpoints, err := env.Kube.ServiceEndpointState(ctx, env.Namespace, pgkube.MysqlDeploymentName(env.FG, state.topo.reader)) + if err != nil { + last = err.Error() + } else { + ready := endpoints.ReadyPodNames("mysql") + last = fmt.Sprintf("ready=%v", ready) + if len(ready) == 0 { + return nil + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + return fmt.Errorf("blocked reader was not shed from its client Service within 60s (last: %s)", last) + }, + } +} + +func s43ReconcileAndRecover(state *s43RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseSettle, + Name: "reconcile errant GTIDs on the primary; convergence restarts the reader", + Do: func(ctx context.Context, env *runner.Env) error { + if err := s43RecoverErrantReader(ctx, env, state); err != nil { + return err + } + state.recovered = true + return nil + }, + } +} + +// s43RecoverErrantReader makes the reader's errant transactions +// non-errant by committing them as empty transactions on the active +// primary (the standard errant-GTID reconciliation), waits for the +// convergence invariant to restart the reader, then drops the scenario +// database through the primary so the errant row itself is removed from +// the reader by replication. +func s43RecoverErrantReader(ctx context.Context, env *runner.Env, state *s43RunState) error { + txns, err := gtidSetTransactions(state.errantSet) + if err != nil { + return fmt.Errorf("parse errant GTID set %q: %w — run `kubectl bloodraven reclone %s` to recover", state.errantSet, err, state.topo.reader) + } + for _, txn := range txns { + if uuid, _, _ := strings.Cut(txn, ":"); !strings.EqualFold(uuid, state.readerUUID) { + return fmt.Errorf("errant set %q contains foreign UUID %q (reader is %s); refusing empty-transaction reconcile — run `kubectl bloodraven reclone %s`", + state.errantSet, uuid, state.readerUUID, state.topo.reader) + } + } + active, err := env.MySQL(state.topo.active) + if err != nil { + return fmt.Errorf("open active mysql: %w", err) + } + var b strings.Builder + for _, txn := range txns { + fmt.Fprintf(&b, "SET GTID_NEXT='%s'; BEGIN; COMMIT; ", txn) + } + b.WriteString("SET GTID_NEXT='AUTOMATIC'") + if _, err := active.Exec(ctx, b.String()); err != nil { + return fmt.Errorf("commit %d empty transactions on %s: %w", len(txns), state.topo.active, err) + } + env.Capture.Note(fmt.Sprintf("committed %d empty transactions on %s to reconcile %s", len(txns), state.topo.active, state.errantSet)) + + waitCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + _, err = env.Wait.UntilCR(waitCtx, env.Namespace, + "convergence invariant restarts the reconciled reader", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return false, "reader status missing", nil + } + err := assertReaderServingStatus(mfg, status, state.topo.activeHost) + return err == nil, fmt.Sprintf("convergence=%s/%s replicating=%v lag=%v", + status.SourceConvergenceState, status.SourceConvergenceReason, status.Replicating, status.SecondsBehindSource), nil + }) + cancel() + if err != nil { + return err + } + + if _, err := active.Exec(ctx, "DROP DATABASE IF EXISTS "+s43DBName); err != nil { + return fmt.Errorf("drop %s through primary: %w", s43DBName, err) + } + return waitReaderClientEndpoint(ctx, env, state.topo.reader, 90*time.Second) +} + +// s43Cleanup retries the errant-GTID reconcile if the settle step never +// ran — a Blocked reader passes the shared baseline (non-promotable +// sites are exempt from the replicating check) and would otherwise leak +// into the next scenario. +func s43Cleanup(state *s43RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + if state.recovered || state.topo.reader == "" { + return nil + } + if state.errantSet == "" { + // Failed before the errant set was computed: either the errant + // write never landed (nothing to do beyond dropping the DB) or + // it landed and must be recomputed here. + active, activeErr := env.MySQL(state.topo.active) + reader, readerErr := env.MySQL(state.topo.reader) + if activeErr != nil || readerErr != nil { + return fmt.Errorf("cleanup: open mysql (active=%v reader=%v)", activeErr, readerErr) + } + readerGtid, err := reader.GtidExecuted(ctx) + if err != nil { + return fmt.Errorf("cleanup: reader gtid: %w", err) + } + activeGtid, err := active.GtidExecuted(ctx) + if err != nil { + return fmt.Errorf("cleanup: active gtid: %w", err) + } + errant, err := active.ScalarString(ctx, "SELECT GTID_SUBTRACT(?, ?)", readerGtid, activeGtid) + if err != nil { + return fmt.Errorf("cleanup: compute errant set: %w", err) + } + state.errantSet = strings.ReplaceAll(strings.TrimSpace(errant), "\n", "") + if state.errantSet == "" { + _, err := active.Exec(ctx, "DROP DATABASE IF EXISTS "+s43DBName) + return err + } + } + return s43RecoverErrantReader(ctx, env, state) + } +} + +// gtidSetTransactions expands a GTID set ("uuid:1-3:7,uuid2:5") into +// individual transactions ("uuid:1", "uuid:2", ...). Errors on an +// implausibly large set so a bad parse can never flood the primary with +// empty transactions. +func gtidSetTransactions(set string) ([]string, error) { + var out []string + for _, uuidPart := range strings.Split(set, ",") { + uuidPart = strings.TrimSpace(uuidPart) + if uuidPart == "" { + continue + } + uuid, intervals, ok := strings.Cut(uuidPart, ":") + if !ok || uuid == "" { + return nil, fmt.Errorf("malformed GTID set element %q", uuidPart) + } + for _, interval := range strings.Split(intervals, ":") { + loRaw, hiRaw, ranged := strings.Cut(interval, "-") + lo, err := strconv.ParseInt(loRaw, 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed GTID interval %q: %w", interval, err) + } + hi := lo + if ranged { + hi, err = strconv.ParseInt(hiRaw, 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed GTID interval %q: %w", interval, err) + } + } + if hi < lo { + return nil, fmt.Errorf("inverted GTID interval %q", interval) + } + for k := lo; k <= hi; k++ { + out = append(out, fmt.Sprintf("%s:%d", uuid, k)) + if len(out) > 50 { + return nil, fmt.Errorf("GTID set expands past 50 transactions; refusing") + } + } + } + } + if len(out) == 0 { + return nil, fmt.Errorf("empty GTID set") + } + return out, nil +} diff --git a/internal/playground/scenarios/s44_reader_source_convergence_invariant.go b/internal/playground/scenarios/s44_reader_source_convergence_invariant.go new file mode 100644 index 0000000..b5b3424 --- /dev/null +++ b/internal/playground/scenarios/s44_reader_source_convergence_invariant.go @@ -0,0 +1,237 @@ +package scenarios + +import ( + "context" + "fmt" + "time" + + v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" + pgkube "github.com/shipstream/bloodraven/internal/playground/kube" + pglogs "github.com/shipstream/bloodraven/internal/playground/logs" + pgmetrics "github.com/shipstream/bloodraven/internal/playground/metrics" + "github.com/shipstream/bloodraven/internal/playground/runner" +) + +const s44MarkerTable = "bloodraven_reader_e2e.convergence_marker" + +func init() { + runner.Register(scenario44ReaderSourceConvergenceInvariant()) +} + +type s44RunState struct { + topo readerTopology + standbyHost string + lastFailoverBefore string + injected bool + recovered bool +} + +// scenario44ReaderSourceConvergenceInvariant is chaos proposal R9 from +// issue #115: direct-source convergence is a periodic invariant of the +// poll loop, not a one-shot switchover event. ANY wrong-source state — +// operator drift, a partially-applied runbook, or a pre-fix chained +// reader left over from an upgrade — heals without a failover. +func scenario44ReaderSourceConvergenceInvariant() runner.Scenario { + state := &s44RunState{} + return runner.Scenario{ + ID: "44-reader-source-convergence-invariant", + Title: "Manually repointed reader converges back to the primary as a poll-loop invariant", + Hypothesis: "Pointing the reader's replication at the promotable standby (a chained topology) is repaired " + + "within a bounded number of poll cycles: the operator logs the documented convergence started/complete " + + "events, repoints the reader directly at the active primary after the GTID containment check, and the " + + "group sees no failover.", + Risk: "low", + DocLink: "playground/chaos-scenarios.md#44-reader-source-convergence-invariant", + Timeout: 6 * time.Minute, + Precheck: s44Precheck(state), + Steps: []runner.Step{ + s44InjectWrongSource(state), + s44ObserveConvergenceLogs(state), + s44VerifyDirectSourceRestored(state), + }, + Cleanup: s44Cleanup(state), + } +} + +func s44Precheck(state *s44RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + *state = s44RunState{} + topo, err := resolveReaderTopology(ctx, env) + if err != nil { + return err + } + state.topo = topo + state.standbyHost = playgroundInternalSiteHost(env.FG, topo.standby, env.Namespace) + return nil + } +} + +func s44InjectWrongSource(state *s44RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseInject, + Name: "repoint the reader's replication at the promotable standby (chained topology)", + Do: func(ctx context.Context, env *runner.Env) error { + if _, err := env.Logs("operator"); err != nil { + return fmt.Errorf("open operator tailer: %w", err) + } + mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) + if err != nil { + return err + } + if mfg.Status.LastFailover != nil { + state.lastFailoverBefore = mfg.Status.LastFailover.Time.UTC().Format(time.RFC3339) + } + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + // SOURCE_HOST is the only changed option; user, password, and + // GTID auto-positioning carry over from the existing channel. + // One batch keeps the stopped window smaller than a poll cycle. + stmt := fmt.Sprintf("STOP REPLICA; CHANGE REPLICATION SOURCE TO SOURCE_HOST='%s'; START REPLICA", state.standbyHost) + if _, err := reader.Exec(ctx, stmt); err != nil { + return fmt.Errorf("repoint reader at standby %s: %w", state.topo.standby, err) + } + state.injected = true + env.Capture.Note(fmt.Sprintf("reader %s now chained through standby %s; expecting invariant repair to %s", + state.topo.reader, state.standbyHost, state.topo.activeHost)) + return nil + }, + } +} + +func s44ObserveConvergenceLogs(state *s44RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseObserve, + Name: "operator logs convergence started (wrong current source) then complete", + Do: func(ctx context.Context, env *runner.Env) error { + tail, err := env.Logs("operator") + if err != nil { + return err + } + startCtx, cancelStart := context.WithTimeout(ctx, 2*time.Minute) + _, err = env.Wait.UntilLog(startCtx, tail, env.StartTime, + "convergence started with currentSource=standby expectedSource=active", + pglogs.Structured("replication source convergence started", map[string]string{ + "site": state.topo.reader, + "currentSource": state.standbyHost, + "expectedSource": state.topo.activeHost, + })) + cancelStart() + if err != nil { + return err + } + completeCtx, cancelComplete := context.WithTimeout(ctx, 2*time.Minute) + _, err = env.Wait.UntilLog(completeCtx, tail, env.StartTime, + "convergence complete back onto the active primary", + pglogs.Structured("replication source convergence complete", map[string]string{ + "site": state.topo.reader, + "source": state.topo.activeHost, + })) + cancelComplete() + return err + }, + } +} + +func s44VerifyDirectSourceRestored(state *s44RunState) runner.Step { + return runner.Step{ + Phase: runner.PhaseVerify, + Name: "reader replicates directly from the primary again; no failover consumed", + Do: func(ctx context.Context, env *runner.Env) error { + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + mfg, err := env.Wait.UntilCR(waitCtx, env.Namespace, + "reader serving status converged on the active primary", + func(mfg *v1alpha1.MysqlFailoverGroup) (bool, string, error) { + status := statusSiteByName(mfg, state.topo.reader) + if status == nil { + return false, "reader status missing", nil + } + err := assertReaderServingStatus(mfg, status, state.topo.activeHost) + return err == nil, fmt.Sprintf("convergence=%s/%s source=%q lag=%v", + status.SourceConvergenceState, status.SourceConvergenceReason, status.SourceHost, status.SecondsBehindSource), nil + }) + cancel() + if err != nil { + return err + } + state.recovered = true + + if ready := pgkube.ReadyCondition(mfg); ready != "True" { + return fmt.Errorf("group Ready=%q after convergence repair, want True", ready) + } + if mfg.Status.ActiveSite != state.topo.active { + return fmt.Errorf("active site changed during convergence repair: %q -> %q", state.topo.active, mfg.Status.ActiveSite) + } + lastFailover := "" + if mfg.Status.LastFailover != nil { + lastFailover = mfg.Status.LastFailover.Time.UTC().Format(time.RFC3339) + } + if lastFailover != state.lastFailoverBefore { + return fmt.Errorf("convergence repair consumed a failover: lastFailover %q -> %q", state.lastFailoverBefore, lastFailover) + } + + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("open reader mysql: %w", err) + } + rs, err := reader.ShowReplicaStatus(ctx) + if err != nil { + return err + } + if !rs.Configured || !rs.IORunning || !rs.SQLRunning || canonicalMySQLHost(rs.SourceHost) != canonicalMySQLHost(state.topo.activeHost) { + return fmt.Errorf("live replica status not direct: configured=%v io=%v sql=%v source=%q, want source %q", + rs.Configured, rs.IORunning, rs.SQLRunning, rs.SourceHost, state.topo.activeHost) + } + + metricCtx, cancelMetric := context.WithTimeout(ctx, 30*time.Second) + defer cancelMetric() + if err := env.Wait.UntilMetric(metricCtx, env.Metrics, + fmt.Sprintf(`replication_source_state{site=%q,state="converged"} == 1`, state.topo.reader), + func(snap *pgmetrics.Snapshot) (bool, string) { + v, ok := snap.Gauge("bloodraven_replication_source_state", map[string]string{"site": state.topo.reader, "state": "converged"}) + return ok && v == 1, fmt.Sprintf("converged=%g(ok=%v)", v, ok) + }, + ); err != nil { + return err + } + + // Write-through proof over the repaired direct channel. + marker := fmt.Sprintf("convergence-%d", time.Now().UnixNano()) + if err := seedMarkerRow(ctx, env, state.topo.active, s44MarkerTable, marker); err != nil { + return err + } + if err := waitForMarkerOnSite(ctx, env, state.topo.reader, s44MarkerTable, marker, 60*time.Second); err != nil { + return err + } + return waitReaderClientEndpoint(ctx, env, state.topo.reader, 90*time.Second) + }, + } +} + +// s44Cleanup restarts the reader's replication if the scenario failed +// mid-way with the channel stopped. The wrong-source state itself needs +// no cleanup — repairing it is exactly what the operator's convergence +// invariant does, and the executor's reconverge wait holds the runner +// until the group is healthy again. +func s44Cleanup(state *s44RunState) func(context.Context, *runner.Env) error { + return func(ctx context.Context, env *runner.Env) error { + if !state.injected || state.recovered { + return nil + } + reader, err := env.MySQL(state.topo.reader) + if err != nil { + return fmt.Errorf("cleanup: open reader mysql: %w", err) + } + rs, err := reader.ShowReplicaStatus(ctx) + if err != nil { + return fmt.Errorf("cleanup: replica status: %w", err) + } + if rs.Configured && (!rs.IORunning || !rs.SQLRunning) { + if _, err := reader.Exec(ctx, "START REPLICA"); err != nil { + return fmt.Errorf("cleanup: start replica: %w", err) + } + } + return nil + } +} diff --git a/playground/chaos-scenarios.md b/playground/chaos-scenarios.md index bcc8a07..9d83375 100644 --- a/playground/chaos-scenarios.md +++ b/playground/chaos-scenarios.md @@ -1080,6 +1080,97 @@ kubectl -n bloodraven-playground patch mysqlfailovergroup playground --type json --- +### 41. Reader Availability During Unplanned Failover {#41-reader-availability-during-unplanned-failover} +**Category**: Reader recovery and endpoint safety | **Risk**: Medium + +Issue #115 chaos proposal R3. + +**Hypothesis**: During the unplanned-failover window the reader keeps serving (stale) reads — staleness is allowed, availability is required. After promotion, source convergence repoints the reader directly at the new primary; at no point does the reader apply events relayed through the demoted primary. + +**Injection**: `make chaos-run SCENARIO=41-reader-availability-during-failover`. Seeds a marker on the active primary and confirms it reaches the reader, then scales the active primary's Deployment to 0 (sustained outage — pod-kill respawns in seconds and tests nothing). + +**Verify**: + +- A continuous observer answers a marker `SELECT` against the reader every 500ms throughout the window. One reconnect is allowed per tick (idle port-forward tunnels can drop); two consecutive failures fail the scenario. At least 20 successful reads are required so the observer provably covered the window. +- `status.activeSite` flips to the promotable standby and nowhere else. +- The reader's `SourceHost` history only ever moves old-primary → new-primary: any third host, or a return to the demoted primary after the repoint, fails the run. +- The reader never enters `SourceConvergenceState=Blocked` or `RecoveryBlocked`. +- Final reader status passes the serving contract against the new primary (`read-only`, replicating, `Converged`, lag ≤ `readOnlyMaxLagSeconds`), a marker written on the new primary replicates to the reader, and the client Service publishes exactly the reader pod again. + +**Cleanup**: The executor restores the old primary's scale; scenario cleanup reuses the scenario-08 auto-reclone path if the returning primary is divergent. + +--- + +### 42. Reader Stall Does Not Degrade the Group {#42-reader-stall-does-not-degrade-the-group} +**Category**: Reader recovery and endpoint safety | **Risk**: Low + +Issue #115 chaos proposal R4. Smoke-profile member: fast, no clone wait. + +**Hypothesis**: A wedged OLAP reader is invisible to the failover machinery — lag grows unbounded with zero group-level effect. The only reactions are endpoint shedding and alertable (not page-able) per-site status and metrics. + +**Injection**: `make chaos-run SCENARIO=42-reader-stall-no-group-degradation`. Applies `CHANGE REPLICATION SOURCE TO SOURCE_DELAY=600` on the reader in a single `STOP REPLICA; ...; START REPLICA` batch. SOURCE_DELAY — not `STOP REPLICA SQL_THREAD` — is the one lag injection the operator will not heal: both threads keep running on the correct source, so convergence sees a converged-but-lagging reader instead of a stopped one (same reasoning as scenario 14). + +**Verify** (soak of 3× `maxLagSeconds`, one primary write per second to grow applied lag): + +- Group `Ready` stays `True` every tick; `status.activeSite` and `status.lastFailover` never change (no failover, no anti-flap cooldown consumed). +- The promotable standby keeps replicating. +- The reader stays `read-only` and never enters `SourceConvergenceState=Blocked` or `RecoveryBlocked`. +- Observed reader lag exceeds `readOnlyMaxLagSeconds` and the client Service sheds the reader endpoint. +- `bloodraven_replication_lag_seconds{site=reader}` exceeds the reader threshold while `bloodraven_replication_source_state{site=reader,state="converged"}` stays 1 — the metrics report the stall honestly. + +**Rollback**: `SOURCE_DELAY=0`; the reader must catch up, pass the serving contract, and rejoin its client Service. Cleanup always clears the delay and drops the soak database through the primary. + +--- + +### 43. Writable Reader Fence {#43-writable-reader-fence} +**Category**: Reader recovery and endpoint safety | **Risk**: Medium + +Issue #115 chaos proposal R5 — the regression gate for role semantics under the worst input (spec gap #2). + +**Hypothesis**: A reader that somehow becomes writable is fenced like a `dr-only` loser without debounce, its errant GTID trips the convergence containment gate instead of a silent repoint, and it is never a promotion target — not even by explicit admin request. + +**Injection**: `make chaos-run SCENARIO=43-writable-reader-fence`. One multi-statement batch on the reader: `super_read_only=OFF`, `read_only=OFF`, then an errant `CREATE DATABASE`/`CREATE TABLE`/`INSERT` (reader-scoped split-brain). Later, `STOP REPLICA` on the reader forces the convergence invariant to act on the diverged follower. + +**Verify**: + +- Operator log contains `fenced writable non-promotable site` with `site=`; `super_read_only` is back `ON` within 30s; group `Ready` and `activeSite` untouched. +- Annotating a planned failover targeting the reader lands `status.plannedFailover.phase=Failed` carrying the "only primary-candidate sites may be promoted" role error, and `bloodraven_planned_failovers_total{target_site=,result="rejected"}` increments. +- After `STOP REPLICA`, the operator logs `replication source convergence blocked`, the reader reaches `SourceConvergenceState=Blocked` with reason `GTIDDiverged` (never a silent restart), `bloodraven_replication_source_state{site=reader,state="blocked"}` flips to 1, and the reader is shed from its client Service. + +**Cleanup**: The scenario reconciles the errant transactions by committing them as empty transactions on the active primary (refusing if the errant set carries any UUID other than the reader's — that demands `kubectl bloodraven reclone`), waits for the convergence invariant to restart the reader, then drops the scenario database through the primary so replication removes the errant row itself. + +--- + +### 44. Reader Source Convergence Invariant {#44-reader-source-convergence-invariant} +**Category**: Reader recovery and endpoint safety | **Risk**: Low + +Issue #115 chaos proposal R9. + +**Hypothesis**: Direct-source convergence is a periodic invariant of the poll loop, not a one-shot switchover event — *any* wrong-source state (operator drift, a partially-applied runbook, a pre-fix chained reader left over from an upgrade) heals with no failover. The divergent-wrong-source counterpart (must go `Blocked`, never silently repoint) is scenario 43's final phase. + +**Injection**: `make chaos-run SCENARIO=44-reader-source-convergence-invariant`. Repoints the reader's replication at the promotable standby (`CHANGE REPLICATION SOURCE TO SOURCE_HOST=` in one batch; credentials and GTID auto-positioning carry over), producing a working chained topology. + +**Verify**: + +- Operator log contains `replication source convergence started` with `site=`, `currentSource=`, `expectedSource=`, followed by `replication source convergence complete` back onto the active primary — the documented log-schema events. +- Reader serving status and live `SHOW REPLICA STATUS` converge back onto the active primary; `bloodraven_replication_source_state{site=reader,state="converged"}` is 1. +- Group `Ready` stays `True`; `status.activeSite` and `status.lastFailover` are untouched. +- A marker written on the primary replicates to the reader over the repaired direct channel, and the client Service publishes exactly the reader pod. + +**Cleanup**: None needed for the wrong-source state itself — repairing it is exactly what the invariant does. Defensive cleanup restarts the reader's replication channel if the scenario failed with it stopped. + +--- + +### Planned reader scenarios (not yet automated) + +Issue #115 proposals R6-R8 remain manual/full-profile follow-ups: + +- **45 (R6) — Bootstrap ordering with reader present**: fresh 3-site deploy must clone the promotable standby and reach failover-capable `Ready=True` *before* the reader clones; a mid-clone reader failure retries without blocking bootstrap. Run manually via `./playground/reset-mysql.sh` + `./playground/setup.sh` while watching `starting bootstrap` ordering in operator logs. +- **46 (R7) — Primary dies mid-reader-clone**: trigger scenario 40's re-clone, hard-kill the donor primary while `CLONE INSTANCE` is in flight; failover must proceed un-wedged and the clone state machine must retry against the new primary. The playground's small dataset makes the in-flight window sub-second, so automation needs an artificial data volume first. +- **47 (R8) — `externalTrafficPolicy: Local` endpoint semantics**: probe the reader NodePort (30306) from reader and non-reader nodes across pod-down/up; requires per-node canary pods and is infra-dependent (kube-proxy implementation). + +--- + ## Execution Plan 1. **Setup**: `k3d cluster create` + `./playground/setup.sh` From 2b76943a86c74d2a4affc896ca143b50b4f1b3e1 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 05:54:30 +0000 Subject: [PATCH 5/7] Wait for planned switchover topology --- internal/playground/scenarios/s02_planned_switchover.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/playground/scenarios/s02_planned_switchover.go b/internal/playground/scenarios/s02_planned_switchover.go index c5eaf71..0a2e745 100644 --- a/internal/playground/scenarios/s02_planned_switchover.go +++ b/internal/playground/scenarios/s02_planned_switchover.go @@ -54,6 +54,8 @@ func verifyFollowersDirectlyFollowNewPrimary() runner.Step { mfg, err := env.Kube.GetMFGNamed(ctx, env.Namespace, env.FG) if err != nil { last = err.Error() + } else if mfg.Status.ActiveSite == "" { + last = "activeSite is temporarily empty while topology reconverges" } else if mfg.Status.ActiveSite != target { return fmt.Errorf("planned failover target changed after success: activeSite=%q want %q", mfg.Status.ActiveSite, target) } else { From dd86c119c6285b316ef2c2e8f1534561aa4c3215 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 06:02:46 +0000 Subject: [PATCH 6/7] Address reader site review findings --- cmd/playground-chaos/reset.go | 40 ++++-- cmd/playground-chaos/reset_test.go | 49 +++++++ docs/docs/monitoring-prometheus.mdx | 12 +- docs/docs/monitoring.mdx | 2 +- internal/controller/reclone_test.go | 30 ++-- internal/controller/reconciler.go | 14 +- internal/controller/reconciler_test.go | 59 ++++++++ internal/controller/runner.go | 21 +-- internal/controller/runner_test.go | 38 +++++ internal/controller/source_convergence.go | 14 +- .../controller/source_convergence_test.go | 23 +++ internal/controller/topology.go | 131 +++++++++++------- internal/controller/topology_test.go | 67 +++++---- internal/controller/updater.go | 18 ++- internal/controller/updater_test.go | 68 ++++++++- internal/metrics/metrics.go | 2 +- internal/metrics/metrics_test.go | 103 +++++++++----- internal/playground/logs/matcher.go | 12 +- internal/playground/logs/matcher_test.go | 24 ++++ internal/state/matrix.go | 22 ++- internal/state/matrix_test.go | 25 +++- playground/chaos-scenarios.md | 4 +- test/envtest/reconciler_test.go | 79 ++++++++--- 23 files changed, 665 insertions(+), 192 deletions(-) diff --git a/cmd/playground-chaos/reset.go b/cmd/playground-chaos/reset.go index db906f1..9c4d4d5 100644 --- a/cmd/playground-chaos/reset.go +++ b/cmd/playground-chaos/reset.go @@ -48,6 +48,11 @@ type resetter struct { runtime string } +type resetPhase struct { + name string + fn func(context.Context, []v1alpha1.SiteSpec) error +} + func resetPlayground(kubeconfig, kctx, namespace, fg, resultsDir string, logger *slog.Logger) int { k, err := loadKube(kubeconfig, kctx, false) if err != nil { @@ -95,10 +100,15 @@ func (r *resetter) run(ctx context.Context) error { return fmt.Errorf("MFG has no spec.sites") } - phases := []struct { - name string - fn func(context.Context, []v1alpha1.SiteSpec) error - }{ + if err := r.runPhases(ctx, sites, r.resetPhases()); err != nil { + return err + } + r.info("reset complete") + return nil +} + +func (r *resetter) resetPhases() []resetPhase { + return []resetPhase{ {"scale operator down", r.scaleOperatorDown}, {"scale MySQL down", r.scaleMysqlDown}, {"sync MySQL deployment specs", r.syncMySQLDeploymentSpecs}, @@ -116,13 +126,15 @@ func (r *resetter) run(ctx context.Context) error { {"wait healthy baseline", r.waitBaseline}, {"normalize fresh baseline", r.normalizeFreshBaseline}, } +} + +func (r *resetter) runPhases(ctx context.Context, sites []v1alpha1.SiteSpec, phases []resetPhase) error { for _, phase := range phases { r.info("%s...", phase.name) if err := phase.fn(ctx, sites); err != nil { return fmt.Errorf("%s: %w", phase.name, err) } } - r.info("reset complete") return nil } @@ -131,16 +143,16 @@ func (r *resetter) run(ctx context.Context) error { // Restarting after the cluster has converged lets the manager discover the // existing primary without recording bootstrap as an operational failover. func (r *resetter) normalizeFreshBaseline(ctx context.Context, sites []v1alpha1.SiteSpec) error { - if err := r.scaleOperatorDown(ctx, sites); err != nil { - return err - } - if err := r.clearMFGStatus(ctx, sites); err != nil { - return err - } - if err := r.startOperator(ctx, sites); err != nil { - return err + return r.runPhases(ctx, sites, r.normalizationPhases()) +} + +func (r *resetter) normalizationPhases() []resetPhase { + return []resetPhase{ + {"scale operator down", r.scaleOperatorDown}, + {"clear MFG status", r.clearMFGStatus}, + {"start operator", r.startOperator}, + {"wait healthy baseline", r.waitBaseline}, } - return r.waitBaseline(ctx, sites) } func (r *resetter) scaleOperatorDown(ctx context.Context, _ []v1alpha1.SiteSpec) error { diff --git a/cmd/playground-chaos/reset_test.go b/cmd/playground-chaos/reset_test.go index 21f7c69..a1fbd25 100644 --- a/cmd/playground-chaos/reset_test.go +++ b/cmd/playground-chaos/reset_test.go @@ -1,6 +1,10 @@ package main import ( + "context" + "errors" + "io" + "log/slog" "reflect" "testing" @@ -11,6 +15,51 @@ import ( v1alpha1 "github.com/shipstream/bloodraven/api/v1alpha1" ) +func TestResetNormalizationPhaseOrdering(t *testing.T) { + r := &resetter{} + resetPhases := r.resetPhases() + if len(resetPhases) < 2 { + t.Fatalf("reset phase count = %d, want at least 2", len(resetPhases)) + } + last := []string{resetPhases[len(resetPhases)-2].name, resetPhases[len(resetPhases)-1].name} + if !reflect.DeepEqual(last, []string{"wait healthy baseline", "normalize fresh baseline"}) { + t.Fatalf("final reset phases = %v, want initial baseline then normalization", last) + } + + normalization := r.normalizationPhases() + var names []string + for _, phase := range normalization { + names = append(names, phase.name) + } + want := []string{"scale operator down", "clear MFG status", "start operator", "wait healthy baseline"} + if !reflect.DeepEqual(names, want) { + t.Fatalf("normalization phases = %v, want %v", names, want) + } +} + +func TestRunResetPhasesStopsOnFirstError(t *testing.T) { + r := &resetter{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + boom := errors.New("boom") + var calls []string + phase := func(name string, err error) resetPhase { + return resetPhase{name: name, fn: func(context.Context, []v1alpha1.SiteSpec) error { + calls = append(calls, name) + return err + }} + } + err := r.runPhases(context.Background(), nil, []resetPhase{ + phase("scale operator down", nil), + phase("clear MFG status", boom), + phase("start operator", nil), + }) + if !errors.Is(err, boom) { + t.Fatalf("runPhases() error = %v, want %v", err, boom) + } + if !reflect.DeepEqual(calls, []string{"scale operator down", "clear MFG status"}) { + t.Fatalf("phase calls = %v, want stop at first error", calls) + } +} + func TestEscapeSQLString(t *testing.T) { cases := []struct { name string diff --git a/docs/docs/monitoring-prometheus.mdx b/docs/docs/monitoring-prometheus.mdx index 73fde2c..eb434c2 100644 --- a/docs/docs/monitoring-prometheus.mdx +++ b/docs/docs/monitoring-prometheus.mdx @@ -89,10 +89,12 @@ In Prometheus, check **Status > Targets** for the `bloodraven` job or ServiceMon ## Reader and source-convergence monitoring -`bloodraven_replication_source_state{site,state}` is a state-set gauge for -every follower. It emits the bounded `state` values `converged`, `pending`, -and `blocked`; exactly one is `1` for a follower and the others are `0`. -Active primaries have no active source state. Combine it with +`bloodraven_replication_source_state{namespace,group,site,state}` is a state-set +gauge for every follower. The `namespace` and `group` labels keep identically +named sites in different failover groups in separate series. It emits the +bounded `state` values `converged`, `pending`, and `blocked`; exactly one is `1` +for a follower and the others are `0`. Active primaries have no active source +state. Combine it with `bloodraven_replication_running{site,thread}` and `bloodraven_replication_lag_seconds{site}` when alerting on a reader. @@ -103,7 +105,7 @@ Degraded. Alert on reader sites directly rather than inferring reader health from group conditions. For example: ```promql -bloodraven_replication_source_state{site="reader",state!="converged"} == 1 +bloodraven_replication_source_state{namespace="warehouse",group="orders",site="reader",state!="converged"} == 1 ``` ```promql diff --git a/docs/docs/monitoring.mdx b/docs/docs/monitoring.mdx index ff066e9..290e7b6 100644 --- a/docs/docs/monitoring.mdx +++ b/docs/docs/monitoring.mdx @@ -55,7 +55,7 @@ Metrics are served on `:8080/metrics` in standard Prometheus exposition format. | `bloodraven_websocket_connected_clients` | Gauge | -- | Number of currently connected WebSocket clients | | `bloodraven_replication_lag_seconds` | Gauge | `site` | Replication lag in seconds on the replica site. `-1` if lag is NULL (not replicating). Only present for replica sites. | | `bloodraven_replication_running` | Gauge | `site`, `thread` | Whether a replication thread is running (`1`=yes, `0`=no). Thread is `io` or `sql`. Only present for replica sites. | -| `bloodraven_replication_source_state` | Gauge | `site`, `state` | Direct-source convergence state set for each follower. `state` is one of `converged`, `pending`, or `blocked`; one series is `1` and the others are `0`. Cleared for the active primary. | +| `bloodraven_replication_source_state` | Gauge | `namespace`, `group`, `site`, `state` | Direct-source convergence state set for each follower, scoped by failover group. `state` is one of `converged`, `pending`, or `blocked`; one series is `1` and the others are `0`. Cleared for the active primary. | | `bloodraven_site_state` | Gauge | `site`, `state` | Current site state as a state-set: `1` for the current state, `0` for others. State is `writable`, `read-only`, `unreachable`, or `unknown`. | | `bloodraven_divergent_transactions` | Gauge | `site` | Number of divergent transactions on a site pending recovery after emergency failover. `0` when healthy. Non-zero means the site has committed transactions that never replicated to the current primary. | | `bloodraven_archiver_upload_failures` | Gauge | `namespace`, `group`, `site` | Cumulative PITR archiver upload failures reported by the per-site sidecar. Monotonic except across sidecar restarts — use `increase()` / `rate()` in dashboards. | diff --git a/internal/controller/reclone_test.go b/internal/controller/reclone_test.go index 3c6c58d..3a0e32d 100644 --- a/internal/controller/reclone_test.go +++ b/internal/controller/reclone_test.go @@ -84,17 +84,25 @@ func TestValidateRecloneRequest_ColdReclone_BareSite_RequiresConfirm(t *testing. } func TestValidateRecloneRequest_ColdReclone_WithConfirm_OK(t *testing.T) { - fg := recloneFG([]string{"iad", "pdx"}, nil) - if err := validateRecloneRequest(fg, RecloneRequest{Site: "iad", GtidPrefix: "confirm=orders"}); err != nil { - t.Errorf("cold reclone with correct confirm token should succeed, got %v", err) - } -} - -func TestValidateRecloneRequest_ReadOnlyRecipient_OK(t *testing.T) { - fg := recloneFG([]string{"iad", "pdx", "reader"}, nil) - fg.Spec.Sites[2].Role = v1alpha1.SiteRoleReadOnly - if err := validateRecloneRequest(fg, RecloneRequest{Site: "reader", GtidPrefix: "confirm=orders"}); err != nil { - t.Errorf("cold reader reclone with correct confirm token should succeed, got %v", err) + tests := []struct { + name string + sites []string + recipient string + role v1alpha1.SiteRole + }{ + {name: "primary candidate", sites: []string{"iad", "pdx"}, recipient: "iad"}, + {name: "read-only reader", sites: []string{"iad", "pdx", "reader"}, recipient: "reader", role: v1alpha1.SiteRoleReadOnly}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fg := recloneFG(tt.sites, nil) + if tt.role != "" { + fg.Spec.SiteByName(tt.recipient).Role = tt.role + } + if err := validateRecloneRequest(fg, RecloneRequest{Site: tt.recipient, GtidPrefix: "confirm=orders"}); err != nil { + t.Errorf("cold reclone with correct confirm token should succeed, got %v", err) + } + }) } } diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go index 7258cdf..8c31557 100644 --- a/internal/controller/reconciler.go +++ b/internal/controller/reconciler.go @@ -721,6 +721,14 @@ func generateMyCnf(fg *v1alpha1.MysqlFailoverGroup, sites ...v1alpha1.SiteSpec) settings[k] = v } + // skip-log-bin / disable-log-bin (underscore spellings are normalized to + // hyphens by normalizedMySQLSettings) would silently defeat the enforced + // log-bin invariant: the sorted render places them after log-bin and + // MySQL honors the last occurrence. Strip both aliases outright so group + // or site overrides cannot reintroduce them. + delete(settings, "skip-log-bin") + delete(settings, "disable-log-bin") + // Build sorted output for deterministic ConfigMap content keys := make([]string, 0, len(settings)) for k := range settings { @@ -1302,7 +1310,8 @@ func (r *MysqlFailoverGroupReconciler) reconcileSiteService(ctx context.Context, if site.IsReadOnlyReader() { selector[labelHealthy] = "yes" } - mutateServiceSpec(svc, effectiveSiteServiceType(fg.Spec.ServiceTemplate, site.ServiceTemplate), effectiveSiteExternalTrafficPolicy(fg.Spec.ServiceTemplate, site.ServiceTemplate), selector, []corev1.ServicePort{ + serviceType := effectiveSiteServiceType(fg.Spec.ServiceTemplate, site.ServiceTemplate) + mutateServiceSpec(svc, serviceType, effectiveSiteExternalTrafficPolicy(fg.Spec.ServiceTemplate, site.ServiceTemplate), selector, []corev1.ServicePort{ { Name: "mysql", Port: mysqlPort, @@ -1311,6 +1320,9 @@ func (r *MysqlFailoverGroupReconciler) reconcileSiteService(ctx context.Context, NodePort: siteServiceNodePort(site.ServiceTemplate), }, }) + if serviceType == corev1.ServiceTypeLoadBalancer { + svc.Spec.LoadBalancerIP = site.LBIP + } svc.Spec.PublishNotReadyAddresses = false return nil }) diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.go index 2b3dca1..d56c1e4 100644 --- a/internal/controller/reconciler_test.go +++ b/internal/controller/reconciler_test.go @@ -683,6 +683,33 @@ func TestGenerateMyCnf_NoCloneDDLTimeout(t *testing.T) { } } +func TestGenerateMyCnf_StripsLogBinDisableAliases(t *testing.T) { + tests := []struct { + name string + key string + }{ + {name: "skip-log-bin hyphen", key: "skip-log-bin"}, + {name: "skip_log_bin underscore", key: "skip_log_bin"}, + {name: "disable-log-bin hyphen", key: "disable-log-bin"}, + {name: "disable_log_bin underscore", key: "disable_log_bin"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fg := newTestFG() + fg.Spec.MysqlConf = map[string]string{tt.key: ""} + + cnf := generateMyCnf(fg) + if strings.Contains(cnf, "skip-log-bin") || strings.Contains(cnf, "disable-log-bin") { + t.Errorf("my.cnf must not contain %s (normalized form); got:\n%s", tt.key, cnf) + } + if !strings.Contains(cnf, "log-bin=/var/lib/mysql/mysql-bin") { + t.Errorf("my.cnf should retain the enforced log-bin setting; got:\n%s", cnf) + } + }) + } +} + func TestGenerateMyCnf_LogBinTrustFunctionCreatorsCanBeOverridden(t *testing.T) { tests := []struct { name string @@ -908,6 +935,38 @@ func TestReaderServicesAndSiteOverrides(t *testing.T) { } } +func TestReaderLoadBalancerIPLifecycle(t *testing.T) { + fg := newTestFG() + reader := v1alpha1.SiteSpec{ + Name: "reader", Role: v1alpha1.SiteRoleReadOnly, LBIP: "203.0.113.30", + ServiceTemplate: &v1alpha1.SiteServiceTemplate{Type: corev1.ServiceTypeLoadBalancer}, + } + r, c := newReconciler(fg) + ctx := context.Background() + if err := r.reconcileSiteService(ctx, fg, reader); err != nil { + t.Fatal(err) + } + var svc corev1.Service + key := types.NamespacedName{Name: "mysql-lion-reader", Namespace: fg.Namespace} + if err := c.Get(ctx, key, &svc); err != nil { + t.Fatal(err) + } + if got := svc.Spec.LoadBalancerIP; got != reader.LBIP { + t.Fatalf("loadBalancerIP = %q, want %q", got, reader.LBIP) + } + + reader.ServiceTemplate.Type = corev1.ServiceTypeClusterIP + if err := r.reconcileSiteService(ctx, fg, reader); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, &svc); err != nil { + t.Fatal(err) + } + if svc.Spec.LoadBalancerIP != "" { + t.Fatalf("loadBalancerIP was not cleared after ClusterIP transition: %q", svc.Spec.LoadBalancerIP) + } +} + func TestMutateServiceSpecLifecycle(t *testing.T) { svc := &corev1.Service{Spec: corev1.ServiceSpec{ ClusterIP: "10.0.0.10", ClusterIPs: []string{"10.0.0.10"}, Type: corev1.ServiceTypeNodePort, diff --git a/internal/controller/runner.go b/internal/controller/runner.go index d2bcdb8..5d92a2a 100644 --- a/internal/controller/runner.go +++ b/internal/controller/runner.go @@ -875,6 +875,19 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam if s.Role == state.SiteRoleReadOnly { continue } + // Evaluate source convergence before the nil-replication guard: a + // failed probe leaves Replication nil while the convergence state is + // Pending/ProbeFailed, and skipping here would suppress the Degraded + // condition. Only read-only followers carry convergence state — the + // writable primary is excluded. + if reason == "Healthy" && s.State == state.StateReadOnly && s.SourceConvergenceState != sourceConvergenceConverged { + setCondition(&freshFG.Status.Conditions, metav1.Condition{ + Type: "Degraded", Status: metav1.ConditionTrue, + ObservedGeneration: freshFG.Generation, LastTransitionTime: now, + Reason: "ReplicationSourceMismatch", + Message: fmt.Sprintf("Replication source on %s is not the active primary", s.Name), + }) + } repl := s.Replication if repl == nil { continue @@ -890,14 +903,6 @@ func (r *TopologyManagerRunner) updateCRStatus(ctx context.Context, nn types.Nam Message: fmt.Sprintf("Replication IO/SQL thread not running on %s", siteName), }) } - if s.SourceConvergenceState != sourceConvergenceConverged { - setCondition(&freshFG.Status.Conditions, metav1.Condition{ - Type: "Degraded", Status: metav1.ConditionTrue, - ObservedGeneration: freshFG.Generation, LastTransitionTime: now, - Reason: "ReplicationSourceMismatch", - Message: fmt.Sprintf("Replication source on %s is not the active primary", siteName), - }) - } if repl.SecondsBehindSource != nil && *repl.SecondsBehindSource > maxLagSeconds { setCondition(&freshFG.Status.Conditions, metav1.Condition{ Type: "Degraded", diff --git a/internal/controller/runner_test.go b/internal/controller/runner_test.go index 63685d9..6447f45 100644 --- a/internal/controller/runner_test.go +++ b/internal/controller/runner_test.go @@ -456,6 +456,44 @@ func TestUpdateCRStatus_WritableNonPromotableIsDegraded(t *testing.T) { } } +func TestUpdateCRStatus_ConvergencePendingWithNilReplicationIsDegraded(t *testing.T) { + fg := newTestFG() + scheme := testScheme() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&v1alpha1.MysqlFailoverGroup{}). + WithObjects(fg).Build() + runner := &TopologyManagerRunner{client: c, logger: testLogger(), managers: make(map[types.NamespacedName]*managedTopology)} + nn := types.NamespacedName{Name: fg.Name, Namespace: fg.Namespace} + + // A failed replica-status probe leaves Replication nil while convergence + // is marked Pending/ProbeFailed. The nil guard must not suppress the + // ReplicationSourceMismatch Degraded condition for the read-only + // follower, and the writable primary must not be flagged. + runner.updateCRStatus(context.Background(), nn, TopologySnapshot{ + Sites: []SiteSnapshot{ + {Name: "dc1", Role: state.SiteRolePrimaryCandidate, State: state.StateWritable}, + {Name: "dc2", Role: state.SiteRolePrimaryCandidate, State: state.StateReadOnly, + Replication: nil, SourceConvergenceState: sourceConvergencePending, SourceConvergenceReason: sourceReasonProbeFailed}, + }, + ActiveSite: "dc1", + DegradedReason: "Healthy", + }) + + var updated v1alpha1.MysqlFailoverGroup + if err := c.Get(context.Background(), nn, &updated); err != nil { + t.Fatal(err) + } + for _, condition := range updated.Status.Conditions { + if condition.Type == "Degraded" && condition.Reason == "ReplicationSourceMismatch" { + if condition.Status != metav1.ConditionTrue { + t.Fatalf("ReplicationSourceMismatch condition = %+v, want True", condition) + } + return + } + } + t.Fatal("ReplicationSourceMismatch Degraded condition not found for probe-failed follower") +} + func TestUpdateOnlyStatusCallbackPreservesNoPrimaryCondition(t *testing.T) { fg := newTestFG() scheme := testScheme() diff --git a/internal/controller/source_convergence.go b/internal/controller/source_convergence.go index 396311e..c7767ac 100644 --- a/internal/controller/source_convergence.go +++ b/internal/controller/source_convergence.go @@ -138,6 +138,12 @@ func (tm *TopologyManager) repointReplica(ctx context.Context, active, follower tm.logger.Info("replication source convergence started", "site", follower.name, "activeSite", active.name, "currentSource", currentSource, "expectedSource", active.host, "fg", tm.cfg.Name) + // Rollback restarts must not reuse the operation context: if the probe or + // source change below consumed its deadline, StartReplica(ctx) would fail + // immediately and leave the unchanged channel stopped. Derive a fresh + // bounded rollback context before stopping replication. + rollbackCtx, rollbackCancel := context.WithTimeout(context.WithoutCancel(ctx), sourceConvergenceOperationTimeout) + defer rollbackCancel() if err := follower.mysql.StopReplica(ctx); err != nil { tm.logSourceFailure(follower, active, "stop", err) return err @@ -146,7 +152,7 @@ func (tm *TopologyManager) repointReplica(ctx context.Context, active, follower // The unchanged channel is safe to resume when the failure was a probe; // divergence deliberately remains stopped for operator review. if !strings.Contains(err.Error(), sourceReasonGTIDDiverged) { - _ = follower.mysql.StartReplica(ctx) + _ = follower.mysql.StartReplica(rollbackCtx) } return err } @@ -154,7 +160,7 @@ func (tm *TopologyManager) repointReplica(ctx context.Context, active, follower Host: active.host, User: tm.bootstrapCfg.ReplUser, Password: tm.bootstrapCfg.ReplPassword, UseSSL: tm.bootstrapCfg.UseSSL, }); err != nil { tm.logSourceFailure(follower, active, "change-source", err) - _ = follower.mysql.StartReplica(ctx) + _ = follower.mysql.StartReplica(rollbackCtx) return err } if err := follower.mysql.StartReplica(ctx); err != nil { @@ -256,7 +262,7 @@ func (tm *TopologyManager) emitSourceConvergenceMetrics() { site := &tm.sites[i] if site.name == active { for _, value := range metrics.AllSourceStates { - metrics.ReplicationSourceState.DeleteLabelValues(site.name, value) + metrics.ReplicationSourceState.DeleteLabelValues(tm.cfg.Namespace, tm.cfg.Name, site.name, value) } continue } @@ -266,7 +272,7 @@ func (tm *TopologyManager) emitSourceConvergenceMetrics() { if current == value { v = 1 } - metrics.ReplicationSourceState.WithLabelValues(site.name, value).Set(v) + metrics.ReplicationSourceState.WithLabelValues(tm.cfg.Namespace, tm.cfg.Name, site.name, value).Set(v) } } } diff --git a/internal/controller/source_convergence_test.go b/internal/controller/source_convergence_test.go index 08ab792..7a12e23 100644 --- a/internal/controller/source_convergence_test.go +++ b/internal/controller/source_convergence_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "errors" + "strings" "testing" "github.com/shipstream/bloodraven/internal/mysql" @@ -86,6 +87,28 @@ func TestSourceConvergence_DivergenceBeforeAndAfterStop(t *testing.T) { }) } +func TestRepointReplica_RollbackUsesFreshBoundedContext(t *testing.T) { + primary := &mockMySQL{gtidExecuted: convergenceTestGTID, respectContext: true} + follower := &mockMySQL{ + readOnly: true, gtidExecuted: convergenceTestGTID, + replicaStatusVal: &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: "wrong"}, + } + tm := newConvergenceManager(t, []state.SiteRole{state.SiteRolePrimaryCandidate, state.SiteRoleReadOnly}, primary, follower) + opCtx, cancel := context.WithCancel(context.Background()) + follower.stopReplicaCancel = cancel + + err := tm.repointReplica(opCtx, &tm.sites[0], &tm.sites[1], "wrong") + if err == nil || !strings.Contains(err.Error(), sourceReasonProbeFailed) { + t.Fatalf("repointReplica() error = %v, want post-stop probe failure", err) + } + if follower.startReplicaCalls != 1 { + t.Fatalf("rollback START REPLICA calls = %d, want 1", follower.startReplicaCalls) + } + if got := follower.startReplicaCtxErrs[0]; got != nil { + t.Fatalf("rollback START REPLICA reused canceled operation context: %v", got) + } +} + func TestSourceConvergence_MutationFailuresAndSuppression(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/controller/topology.go b/internal/controller/topology.go index b514bc6..303b97c 100644 --- a/internal/controller/topology.go +++ b/internal/controller/topology.go @@ -17,6 +17,14 @@ import ( "github.com/shipstream/bloodraven/internal/util" ) +const mysqlSafetyProbeTimeout = 5 * time.Second + +func withMySQLSafetyTimeout(ctx context.Context, probe func(context.Context) error) error { + probeCtx, cancel := context.WithTimeout(ctx, mysqlSafetyProbeTimeout) + defer cancel() + return probe(probeCtx) +} + // SiteTopologyConfig holds per-site configuration for the topology manager. type SiteTopologyConfig struct { Name string @@ -1293,7 +1301,10 @@ func (tm *TopologyManager) fenceWritableNonPromotableSites(ctx context.Context) continue } attempted = true - if err := site.mysql.SetSuperReadOnly(ctx, true); err != nil { + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + return site.mysql.SetSuperReadOnly(probeCtx, true) + }) + if err != nil { tm.logger.Error("failed to fence writable non-promotable site", "site", site.name, "role", site.role, "error", err) } else { tm.logger.Warn("fenced writable non-promotable site", "site", site.name, "role", site.role) @@ -1383,7 +1394,10 @@ func (tm *TopologyManager) applyCrossSiteAction(ctx context.Context, action stat continue } tm.logger.Info("fencing returning old primary (split brain after failover)", "site", tm.sites[i].name) - if err := tm.sites[i].mysql.SetSuperReadOnly(ctx, true); err != nil { + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + return tm.sites[i].mysql.SetSuperReadOnly(probeCtx, true) + }) + if err != nil { tm.logger.Error("failed to fence returning old primary", "site", tm.sites[i].name, "error", err) } } @@ -1411,7 +1425,10 @@ func (tm *TopologyManager) applyCrossSiteAction(ctx context.Context, action stat if site := tm.getSite(loser); site != nil && site.state == state.StateWritable { tm.logger.Warn("split-brain auto-resolve: fencing non-preferred site per spec.splitBrainPolicy.sitePriorities", "winner", winner, "fencedSite", loser) - if err := site.mysql.SetSuperReadOnly(ctx, true); err != nil { + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + return site.mysql.SetSuperReadOnly(probeCtx, true) + }) + if err != nil { tm.logger.Error("failed to fence non-preferred site", "site", loser, "error", err) } else { metrics.SplitBrainAutoResolveTotal.WithLabelValues(winner).Inc() @@ -2065,34 +2082,35 @@ func (tm *TopologyManager) detectEmptySite(ctx context.Context) (donor, empty st (site.state != state.StateWritable && site.state != state.StateReadOnly) { continue } - rs, probeErr := site.mysql.ShowReplicaStatus(ctx) - if probeErr != nil { - if site.role == state.SiteRoleReadOnly { - continue + var rs *mysql.ReplicaStatus + var gtid mysql.GTIDSet + var hasSchemas bool + probeErr := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + var err error + rs, err = site.mysql.ShowReplicaStatus(probeCtx) + if err != nil { + return err } - return "", "" - } - raw, probeErr := site.mysql.GetGtidExecuted(ctx) + raw, err := site.mysql.GetGtidExecuted(probeCtx) + if err != nil { + return err + } + gtid, err = mysql.ParseGTIDSet(raw) + if err != nil { + return err + } + hasSchemas = !gtid.IsEmpty() + if checker, ok := site.mysql.(userSchemaChecker); ok { + hasSchemas, err = checker.HasUserSchemas(probeCtx) + } + return err + }) if probeErr != nil { if site.role == state.SiteRoleReadOnly { continue } return "", "" } - gtid, probeErr := mysql.ParseGTIDSet(raw) - if probeErr != nil { - return "", "" - } - hasSchemas := !gtid.IsEmpty() - if checker, ok := site.mysql.(userSchemaChecker); ok { - hasSchemas, probeErr = checker.HasUserSchemas(ctx) - if probeErr != nil { - if site.role == state.SiteRoleReadOnly { - continue - } - return "", "" - } - } freshInitialized := site.state == state.StateReadOnly && !hasSchemas if rs == nil && (gtid.IsEmpty() || freshInitialized) { return active.name, site.name @@ -2109,24 +2127,30 @@ func (tm *TopologyManager) detectAndFenceEmptyWritableSite(ctx context.Context) if site.state != state.StateWritable || site.role == state.SiteRoleReadOnly { continue } - rs, err := site.mysql.ShowReplicaStatus(ctx) - if err != nil || rs != nil { - return "", "" - } - raw, err := site.mysql.GetGtidExecuted(ctx) - if err != nil { - return "", "" - } - gtid, err := mysql.ParseGTIDSet(raw) - if err != nil { - return "", "" - } - hasSchemas := !gtid.IsEmpty() - if checker, ok := site.mysql.(userSchemaChecker); ok { - hasSchemas, err = checker.HasUserSchemas(ctx) + var rs *mysql.ReplicaStatus + var hasSchemas bool + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + var err error + rs, err = site.mysql.ShowReplicaStatus(probeCtx) + if err != nil || rs != nil { + return err + } + raw, err := site.mysql.GetGtidExecuted(probeCtx) if err != nil { - return "", "" + return err + } + gtid, err := mysql.ParseGTIDSet(raw) + if err != nil { + return err + } + hasSchemas = !gtid.IsEmpty() + if checker, ok := site.mysql.(userSchemaChecker); ok { + hasSchemas, err = checker.HasUserSchemas(probeCtx) } + return err + }) + if err != nil || rs != nil { + return "", "" } if hasSchemas { if donor != nil || !site.isPromotable() { @@ -2143,10 +2167,15 @@ func (tm *TopologyManager) detectAndFenceEmptyWritableSite(ctx context.Context) if donor == nil || recipient == nil || donor.host == "" { return "", "" } - if err := recipient.mysql.SetSuperReadOnly(ctx, true); err != nil { - return "", "" - } - readOnly, err := recipient.mysql.CheckReadOnly(ctx) + var readOnly bool + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + if err := recipient.mysql.SetSuperReadOnly(probeCtx, true); err != nil { + return err + } + var err error + readOnly, err = recipient.mysql.CheckReadOnly(probeCtx) + return err + }) if err != nil || !readOnly || tm.confirmWritable(ctx, donor) != nil { return "", "" } @@ -2193,12 +2222,20 @@ func (tm *TopologyManager) startBootstrap(ctx context.Context) { if tm.sites[i].name == seed { continue } - if err := tm.sites[i].mysql.SetSuperReadOnly(ctx, true); err != nil { + var readOnly bool + err := withMySQLSafetyTimeout(ctx, func(probeCtx context.Context) error { + if err := tm.sites[i].mysql.SetSuperReadOnly(probeCtx, true); err != nil { + return err + } + var err error + readOnly, err = tm.sites[i].mysql.CheckReadOnly(probeCtx) + return err + }) + if err != nil { tm.logger.Error("fresh-deploy bootstrap: failed to fence non-seed site", "site", tm.sites[i].name, "error", err) return } - readOnly, err := tm.sites[i].mysql.CheckReadOnly(ctx) - if err != nil || !readOnly { + if !readOnly { tm.logger.Error("fresh-deploy bootstrap: non-seed fence not confirmed", "site", tm.sites[i].name, "error", err) return } diff --git a/internal/controller/topology_test.go b/internal/controller/topology_test.go index eecb54d..65a28cf 100644 --- a/internal/controller/topology_test.go +++ b/internal/controller/topology_test.go @@ -20,29 +20,33 @@ import ( // --- Mock MySQL --- type mockMySQL struct { - mu sync.Mutex - readOnly bool - err error - promoted bool - replicaStatusVal *mysql.ReplicaStatus - replicaStatusErr error - gtidExecuted string - gtidExecutedErr error - hasUserSchemas *bool - userSchemasErr error - stopReplicaCalls int - resetReplicaCalls int - changeSourceCalls int - startReplicaCalls int - stopReplicaErr error - changeSourceErr error - startReplicaErr error - gtidSequence []string - gtidCalls int - changeDoesNotUpdate bool - superReadOnlyCalls int - superReadOnlyErr error - clonePrimaryHost string + mu sync.Mutex + readOnly bool + err error + promoted bool + replicaStatusVal *mysql.ReplicaStatus + replicaStatusErr error + gtidExecuted string + gtidExecutedErr error + hasUserSchemas *bool + userSchemasErr error + stopReplicaCalls int + resetReplicaCalls int + changeSourceCalls int + startReplicaCalls int + stopReplicaErr error + stopReplicaCancel context.CancelFunc + changeSourceErr error + startReplicaErr error + startReplicaCtxErrs []error + respectContext bool + gtidSequence []string + gtidCalls int + changeDoesNotUpdate bool + superReadOnlyCalls int + superReadOnlyErr error + superReadOnlyHadDeadline bool + clonePrimaryHost string } func testBoolPtr(v bool) *bool { @@ -65,10 +69,11 @@ func (m *mockMySQL) Promote(_ context.Context) error { func (m *mockMySQL) Close() error { return nil } -func (m *mockMySQL) SetSuperReadOnly(_ context.Context, on bool) error { +func (m *mockMySQL) SetSuperReadOnly(ctx context.Context, on bool) error { m.mu.Lock() defer m.mu.Unlock() m.superReadOnlyCalls++ + _, m.superReadOnlyHadDeadline = ctx.Deadline() if m.superReadOnlyErr != nil { return m.superReadOnlyErr } @@ -94,6 +99,9 @@ func TestReadOnlyRoleSafety(t *testing.T) { if !tm.fenceWritableNonPromotableSites(context.Background()) || reader.superReadOnlyCalls != 1 { t.Fatalf("reader fence calls = %d", reader.superReadOnlyCalls) } + if !reader.superReadOnlyHadDeadline { + t.Fatal("reader fencing call did not receive a bounded context") + } taint := true tm.applyPerSiteAction(context.Background(), &tm.sites[2], state.Action{Taint: &taint}) if got := len(tm.tainter.(*mockTainter).taints); got != 0 { @@ -208,6 +216,9 @@ func (m *mockMySQL) StopReplica(_ context.Context) error { m.mu.Lock() defer m.mu.Unlock() m.stopReplicaCalls++ + if m.stopReplicaCancel != nil { + m.stopReplicaCancel() + } return m.stopReplicaErr } func (m *mockMySQL) ResetReplicaAll(_ context.Context) error { @@ -243,10 +254,11 @@ func (m *mockMySQL) ChangeReplicationSource(_ context.Context, opts mysql.Replic m.replicaStatusVal.SourceHost = opts.Host return nil } -func (m *mockMySQL) StartReplica(_ context.Context) error { +func (m *mockMySQL) StartReplica(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() m.startReplicaCalls++ + m.startReplicaCtxErrs = append(m.startReplicaCtxErrs, ctx.Err()) if m.startReplicaErr != nil { return m.startReplicaErr } @@ -260,9 +272,12 @@ func (m *mockMySQL) StartReplicaSQLThread(_ context.Context) error { return nil func (m *mockMySQL) WaitForRelayLogDrain(_ context.Context, _ time.Duration) error { return nil } -func (m *mockMySQL) GetGtidExecuted(_ context.Context) (string, error) { +func (m *mockMySQL) GetGtidExecuted(ctx context.Context) (string, error) { m.mu.Lock() defer m.mu.Unlock() + if m.respectContext && ctx.Err() != nil { + return "", ctx.Err() + } if len(m.gtidSequence) > 0 { idx := m.gtidCalls if idx >= len(m.gtidSequence) { diff --git a/internal/controller/updater.go b/internal/controller/updater.go index 668367e..4eb70db 100644 --- a/internal/controller/updater.go +++ b/internal/controller/updater.go @@ -209,11 +209,20 @@ func (u *UpdateController) ExecuteTargets(ctx context.Context, active UpdateTarg return processed, nil } + processedFollowers := make(map[string]struct{}, len(processed)) + for _, name := range processed { + processedFollowers[name] = struct{}{} + } var handoff *UpdateTarget for i := range followers { if !followers[i].Promotable { continue } + if followers[i].Drifted { + if _, updated := processedFollowers[followers[i].Name]; !updated { + continue + } + } if err := u.requireDirectReplica(ctx, followers[i]); err == nil { handoff = &followers[i] break @@ -231,13 +240,16 @@ func (u *UpdateController) ExecuteTargets(ctx context.Context, active UpdateTarg if err != nil { return processed, fmt.Errorf("failover during update: %w", err) } + // failover.Execute has already moved MySQL authority. Record that change + // before the confirmation probe so a transient probe failure cannot leave + // controller status pointing at the old primary. + if onPromoted != nil { + onPromoted(handoff.Name, promotionGTID) + } readOnly, err := handoff.Checker.CheckReadOnly(ctx) if err != nil || readOnly { return processed, fmt.Errorf("updated standby %s was not confirmed writable after handoff", handoff.Name) } - if onPromoted != nil { - onPromoted(handoff.Name, promotionGTID) - } u.setPhase(UpdatePhaseUpdateOldPrimary) if err := applyUpdate(ctx, active.Name); err != nil { return processed, fmt.Errorf("update old active %s: %w", active.Name, err) diff --git a/internal/controller/updater_test.go b/internal/controller/updater_test.go index 5e69aec..9e35abf 100644 --- a/internal/controller/updater_test.go +++ b/internal/controller/updater_test.go @@ -128,6 +128,57 @@ func TestUpdateController_ExecuteTargetsSkipsUnhealthyReader(t *testing.T) { } } +func TestUpdateController_ExecuteTargetsDoesNotHandoffSkippedDriftedFollower(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + standby := &flappingChecker{failFirst: true, sourceHost: "primary-internal"} + var applied []string + _, err := uc.ExecuteTargets(context.Background(), UpdateTarget{ + Name: "primary", Checker: &testutil.FakeMySQL{ReadOnlyVal: false}, Drifted: true, + }, []UpdateTarget{{ + Name: "standby", Host: "standby-internal", Checker: standby, Promotable: true, + Drifted: true, ExpectedSource: "primary-internal", + }}, func(_ context.Context, name string) error { + applied = append(applied, name) + return nil + }, nil) + if err == nil || !strings.Contains(err.Error(), "no healthy promotable standby") { + t.Fatalf("ExecuteTargets() error = %v, want no healthy standby", err) + } + if standby.tick != 1 { + t.Fatalf("skipped drifted standby was reprobed for handoff: probes=%d", standby.tick) + } + if len(applied) != 0 { + t.Fatalf("skipped drifted standby or active site was updated: %v", applied) + } +} + +func TestUpdateController_ExecuteTargetsRecordsPromotionBeforeConfirmation(t *testing.T) { + logger := testutil.TestLogger() + uc := NewUpdateController(NewFailoverController(logger), logger) + standby := &postPromotionProbeFailureChecker{FakeMySQL: &testutil.FakeMySQL{ + ReadOnlyVal: true, + ReplicaStatusVal: &mysql.ReplicaStatus{ + IORunning: true, SQLRunning: true, SourceHost: "primary-internal", + }, + }} + promoted := "" + _, err := uc.ExecuteTargets(context.Background(), UpdateTarget{ + Name: "primary", Checker: &testutil.FakeMySQL{ReadOnlyVal: false}, Drifted: true, + }, []UpdateTarget{{ + Name: "standby", Host: "standby-internal", Checker: standby, Promotable: true, + ExpectedSource: "primary-internal", + }}, func(context.Context, string) error { return nil }, func(site, _ string) { + promoted = site + }) + if err == nil || !strings.Contains(err.Error(), "not confirmed writable") { + t.Fatalf("ExecuteTargets() error = %v, want confirmation failure", err) + } + if promoted != "standby" { + t.Fatalf("promotion callback site = %q, want standby", promoted) + } +} + func TestUpdateController_ExecuteTargetsActiveLastRequiresDirectReplica(t *testing.T) { logger := testutil.TestLogger() uc := NewUpdateController(NewFailoverController(logger), logger) @@ -745,6 +796,8 @@ type flappingChecker struct { mu sync.Mutex writable bool errorEvery int + failFirst bool + sourceHost string tick int } @@ -752,14 +805,14 @@ func (f *flappingChecker) CheckReadOnly(_ context.Context) (bool, error) { f.mu.Lock() defer f.mu.Unlock() f.tick++ - if f.errorEvery > 0 && f.tick%f.errorEvery == 0 { + if (f.failFirst && f.tick == 1) || (f.errorEvery > 0 && f.tick%f.errorEvery == 0) { return false, errors.New("conn refused") } return !f.writable, nil } func (f *flappingChecker) ShowReplicaStatus(_ context.Context) (*mysql.ReplicaStatus, error) { - return &mysql.ReplicaStatus{}, nil + return &mysql.ReplicaStatus{IORunning: true, SQLRunning: true, SourceHost: f.sourceHost}, nil } func (f *flappingChecker) Promote(_ context.Context) error { return nil } @@ -784,6 +837,17 @@ func (f *flappingChecker) CloneInstance(_ context.Context, _, _, _ string, _ boo } func (f *flappingChecker) Close() error { return nil } +type postPromotionProbeFailureChecker struct { + *testutil.FakeMySQL +} + +func (c *postPromotionProbeFailureChecker) CheckReadOnly(ctx context.Context) (bool, error) { + if !c.ReadOnlyVal { + return false, errors.New("post-promotion probe failed") + } + return c.FakeMySQL.CheckReadOnly(ctx) +} + type startableReplicaChecker struct { starts int } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 40a531d..9cc78a6 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -97,7 +97,7 @@ var ( ReplicationSourceState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "bloodraven_replication_source_state", Help: "Direct-primary replication source convergence as a state-set.", - }, []string{"site", "state"}) + }, []string{"namespace", "group", "site", "state"}) SiteState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "bloodraven_site_state", diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 9612da2..fa8b39a 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -6,49 +6,80 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func TestRegisterIncludesRestoreMetrics(t *testing.T) { +func TestReplicationSourceStateSeparatesFailoverGroups(t *testing.T) { reg := prometheus.NewRegistry() - Register(reg) - RestoreDurationSeconds.WithLabelValues("ns", "group", "init_from_backup", "iad").Observe(1) - RestoreLastSuccessTimestamp.WithLabelValues("ns", "group", "init_from_backup", "iad").Set(1) - RestoreLastSourceSizeBytes.WithLabelValues("ns", "group", "init_from_backup", "iad").Set(1) - defer RestoreDurationSeconds.DeleteLabelValues("ns", "group", "init_from_backup", "iad") - defer RestoreLastSuccessTimestamp.DeleteLabelValues("ns", "group", "init_from_backup", "iad") - defer RestoreLastSourceSizeBytes.DeleteLabelValues("ns", "group", "init_from_backup", "iad") + reg.MustRegister(ReplicationSourceState) + ReplicationSourceState.WithLabelValues("ns-a", "orders", "reader", "converged").Set(1) + ReplicationSourceState.WithLabelValues("ns-b", "orders", "reader", "converged").Set(1) + defer ReplicationSourceState.DeleteLabelValues("ns-a", "orders", "reader", "converged") + defer ReplicationSourceState.DeleteLabelValues("ns-b", "orders", "reader", "converged") + families, err := reg.Gather() if err != nil { - t.Fatalf("gather: %v", err) - } - want := map[string]bool{ - "bloodraven_restore_duration_seconds": false, - "bloodraven_restore_last_success_timestamp_seconds": false, - "bloodraven_restore_last_source_size_bytes": false, - } - for _, mf := range families { - if _, ok := want[mf.GetName()]; ok { - want[mf.GetName()] = true - } + t.Fatal(err) } - for name, found := range want { - if !found { - t.Fatalf("metric %s was not registered", name) - } + if len(families) != 1 || len(families[0].Metric) != 2 { + t.Fatalf("replication source series = %v, want two distinct group-scoped series", families) } } -func TestRegisterIncludesReplicationSourceState(t *testing.T) { - reg := prometheus.NewRegistry() - Register(reg) - ReplicationSourceState.WithLabelValues("reader", "converged").Set(1) - defer ReplicationSourceState.DeleteLabelValues("reader", "converged") - families, err := reg.Gather() - if err != nil { - t.Fatal(err) +func TestRegisterIncludesMetrics(t *testing.T) { + tests := []struct { + name string + familyName string + seed func() func() + }{ + { + name: "restore duration", familyName: "bloodraven_restore_duration_seconds", + seed: func() func() { + labels := []string{"ns", "group", "init_from_backup", "iad"} + RestoreDurationSeconds.WithLabelValues(labels...).Observe(1) + return func() { RestoreDurationSeconds.DeleteLabelValues(labels...) } + }, + }, + { + name: "restore success", familyName: "bloodraven_restore_last_success_timestamp_seconds", + seed: func() func() { + labels := []string{"ns", "group", "init_from_backup", "iad"} + RestoreLastSuccessTimestamp.WithLabelValues(labels...).Set(1) + return func() { RestoreLastSuccessTimestamp.DeleteLabelValues(labels...) } + }, + }, + { + name: "restore source size", familyName: "bloodraven_restore_last_source_size_bytes", + seed: func() func() { + labels := []string{"ns", "group", "init_from_backup", "iad"} + RestoreLastSourceSizeBytes.WithLabelValues(labels...).Set(1) + return func() { RestoreLastSourceSizeBytes.DeleteLabelValues(labels...) } + }, + }, + { + name: "replication source state", familyName: "bloodraven_replication_source_state", + seed: func() func() { + labels := []string{"ns", "group", "reader", "converged"} + ReplicationSourceState.WithLabelValues(labels...).Set(1) + return func() { ReplicationSourceState.DeleteLabelValues(labels...) } + }, + }, } - for _, family := range families { - if family.GetName() == "bloodraven_replication_source_state" { - return - } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reg := prometheus.NewRegistry() + Register(reg) + cleanup := tt.seed() + defer cleanup() + + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + for _, family := range families { + if family.GetName() == tt.familyName { + return + } + } + t.Fatalf("metric %s was not registered", tt.familyName) + }) } - t.Fatal("replication source state metric was not registered") } diff --git a/internal/playground/logs/matcher.go b/internal/playground/logs/matcher.go index b93a407..8c1637d 100644 --- a/internal/playground/logs/matcher.go +++ b/internal/playground/logs/matcher.go @@ -66,8 +66,18 @@ func Structured(msg string, fields map[string]string) Predicate { } } +// textFieldMatches reports whether line contains the complete +// whitespace-delimited slog token key=value (or key="value"). Substring +// matching is not sufficient: key boundaries (oldsource=...) and value +// prefixes (source=auto-clone-old for source=auto-clone) would otherwise +// falsely satisfy chaos log assertions. func textFieldMatches(line, key, value string) bool { - return strings.Contains(line, key+"="+value) || strings.Contains(line, key+"="+strconv.Quote(value)) + pattern := `(^|[[:space:]])` + + regexp.QuoteMeta(key) + `=(` + + regexp.QuoteMeta(value) + `|` + + regexp.QuoteMeta(strconv.Quote(value)) + + `)($|[[:space:]])` + return regexp.MustCompile(pattern).MatchString(line) } // Wait blocks until a line matching the predicate is observed in the diff --git a/internal/playground/logs/matcher_test.go b/internal/playground/logs/matcher_test.go index 678b3bb..722df7c 100644 --- a/internal/playground/logs/matcher_test.go +++ b/internal/playground/logs/matcher_test.go @@ -62,3 +62,27 @@ func TestStructuredMatchesJSONAndText(t *testing.T) { t.Error("Structured predicate matched the wrong source") } } + +func TestStructuredRejectsPartialTextTokens(t *testing.T) { + pred := Structured("starting bootstrap", map[string]string{ + "source": "auto-clone", + "recipient": "reader", + }) + for _, line := range []string{ + // Value prefix must not satisfy the exact value. + `msg="starting bootstrap" source=auto-clone-old recipient=reader`, + // Key prefix must not satisfy the exact key. + `msg="starting bootstrap" oldsource=auto-clone recipient=reader`, + // A matching value on a different key must not leak across fields. + `msg="starting bootstrap" source=auto-clone recipient=reader2`, + } { + if pred(line) { + t.Errorf("Structured predicate matched partial token in %q", line) + } + } + // Quoted values with spaces still match as a single token. + quoted := Structured("replication source convergence complete", map[string]string{"site": "reader"}) + if !quoted(`time=now level=INFO msg="replication source convergence complete" site=reader`) { + t.Error("Structured predicate rejected quoted msg containing spaces") + } +} diff --git a/internal/state/matrix.go b/internal/state/matrix.go index 079f6e6..12cd75e 100644 --- a/internal/state/matrix.go +++ b/internal/state/matrix.go @@ -69,7 +69,14 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros var action CrossSiteAction var writable, readOnly, unreachable []SiteObservation + coreCount := 0 for _, obs := range observations { + // Count configured core sites by role before state classification so an + // Unknown observation remains part of the topology instead of making an + // all-unknown startup look like an empty, healthy group. + if obs.Role != SiteRoleReadOnly { + coreCount++ + } if obs.State == StateWritable && obs.Role != SiteRolePrimaryCandidate { action.FenceSites = append(action.FenceSites, obs.Name) continue @@ -87,7 +94,6 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros } } - coreCount := len(writable) + len(readOnly) + len(unreachable) if coreCount == 0 { if len(action.FenceSites) > 0 { action.Alert = fmt.Sprintf("writable non-promotable site requires fencing (%s)", strings.Join(action.FenceSites, ", ")) @@ -98,6 +104,15 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros return action } + // Fencing is best-effort and must be confirmed by a later poll before any + // promotion. Otherwise a failed fence and a successful promotion could + // leave two sites accepting writes. + if len(action.FenceSites) > 0 { + action.Alert = fmt.Sprintf("writable non-promotable site requires fencing (%s)", strings.Join(action.FenceSites, ", ")) + action.Reason = "Degraded" + return action + } + // Total loss: every site is unreachable. if len(unreachable) == coreCount { action.Alert = "TOTAL LOSS: all sites are unreachable" @@ -147,11 +162,6 @@ func EvalCrossSite(observations []SiteObservation, sitePriorities []string) Cros action.Reason = "Degraded" return action } - if len(action.FenceSites) > 0 { - action.Alert = fmt.Sprintf("writable non-promotable site requires fencing (%s)", strings.Join(action.FenceSites, ", ")) - action.Reason = "Degraded" - return action - } action.Reason = "Healthy" return action } diff --git a/internal/state/matrix_test.go b/internal/state/matrix_test.go index 31d794a..fd892dd 100644 --- a/internal/state/matrix_test.go +++ b/internal/state/matrix_test.go @@ -57,6 +57,12 @@ func TestEvalCrossSite_TwoSite(t *testing.T) { wantAlert: true, wantReason: "TotalLoss", }, + { + name: "all unknown candidates are not healthy", + observations: []SiteObservation{obs("iad", pc, StateUnknown), obs("pdx", pc, StateUnknown)}, + wantAlert: true, + wantReason: "NoPrimary", + }, } for _, tt := range tests { @@ -191,21 +197,29 @@ func TestEvalCrossSite_ReadOnlyReaderIsolation(t *testing.T) { wantFence: []string{"dr"}, }, { - name: "sole writable reader is not a primary", + name: "sole writable reader is fenced before no-primary handling", obs: []SiteObservation{ obs("iad", pc, StateReadOnly), obs("pdx", pc, StateReadOnly), obs("reader", reader, StateWritable), }, - wantReason: "NoPrimary", + wantReason: "Degraded", wantFence: []string{"reader"}, }, { - name: "writable dr only is fenced and not primary", + name: "writable dr only is fenced before no-primary handling", obs: []SiteObservation{ obs("iad", pc, StateReadOnly), obs("pdx", pc, StateReadOnly), obs("dr", dr, StateWritable), }, - wantReason: "NoPrimary", + wantReason: "Degraded", wantFence: []string{"dr"}, }, + { + name: "writable reader blocks promotion until fence is confirmed", + obs: []SiteObservation{ + obs("iad", pc, StateUnreachable), obs("pdx", pc, StateReadOnly), obs("reader", reader, StateWritable), + }, + wantReason: "Degraded", + wantFence: []string{"reader"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -216,6 +230,9 @@ func TestEvalCrossSite_ReadOnlyReaderIsolation(t *testing.T) { if got.SplitBrain { t.Fatal("non-promotable writable site must not create split brain") } + if len(got.FenceSites) > 0 && len(got.PromotionCandidates) > 0 { + t.Fatalf("fencing and promotion must not be emitted together: %+v", got) + } }) } } diff --git a/playground/chaos-scenarios.md b/playground/chaos-scenarios.md index 9d83375..5c84c47 100644 --- a/playground/chaos-scenarios.md +++ b/playground/chaos-scenarios.md @@ -277,7 +277,7 @@ kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx mysql- kubectl -n $NS scale deployment mysql-playground-iad mysql-playground-pdx mysql-playground-reader --replicas=1 ``` -**Verify**: Operator logs "TOTAL LOSS: both sites are unreachable" during outage. After recovery, one site writable, one read-only, replication running. No operator crash or permanent degraded state. +**Verify**: Operator logs a total-loss alert during the outage. After recovery, one site is writable and two followers are read-only with replication running. No operator crash or permanent degraded state. --- @@ -1070,7 +1070,7 @@ kubectl -n bloodraven-playground patch mysqlfailovergroup playground --type json - Exactly one spec site has effective role `read-only`; its pod is on a dedicated `zone-reader` worker. - A marker written to the active primary is visible on the reader before loss and after recovery. -- A continuous observer fails immediately if the group `Ready` condition is ever not `True` during scale-down, empty-datadir detection, clone, or catch-up. +- A continuous observer records any group `Ready` invariant violation during scale-down, empty-datadir detection, clone, or catch-up; `stopAndCheck` reports the first recorded error before the step completes. - Operator logs contain `starting bootstrap` with `source=auto-clone`, `donor=`, `recipient=reader`, and `donorHost=mysql-playground--internal.bloodraven-playground.svc.cluster.local`. - The active site never changes. Final reader status is `State=read-only`, `SourceConvergenceState=Converged`, direct `SourceHost`, known lag at or below `readOnlyMaxLagSeconds`, and live MySQL IO/SQL threads running. - The reader client Service exposes only MySQL and has no serving EndpointSlice target while unhealthy. After convergence it has exactly the replacement reader pod. diff --git a/test/envtest/reconciler_test.go b/test/envtest/reconciler_test.go index fbc77c0..6a9715c 100644 --- a/test/envtest/reconciler_test.go +++ b/test/envtest/reconciler_test.go @@ -4,11 +4,13 @@ package envtest import ( "fmt" + "strings" "testing" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -66,9 +68,8 @@ func newTestFG(namespace string) *v1alpha1.MysqlFailoverGroup { func ensureNamespace(t *testing.T, name string) { t.Helper() ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - if err := k8sClient.Create(ctx, ns); err != nil { - // Ignore already-exists errors. - return + if err := k8sClient.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("create namespace %s: %v", name, err) } } @@ -168,8 +169,13 @@ func TestEnvtest_ReadOnlyRoleValidation(t *testing.T) { if tt.wantOK && err != nil { t.Fatalf("Create() rejected valid object: %v", err) } - if !tt.wantOK && err == nil { - t.Fatal("Create() accepted invalid object") + if !tt.wantOK { + if err == nil { + t.Fatal("Create() accepted invalid object") + } + if !apierrors.IsInvalid(err) { + t.Fatalf("Create() error = %T %v, want Kubernetes Invalid schema error", err, err) + } } }) } @@ -372,8 +378,8 @@ func TestEnvtest_ReconcilerCreatesResources(t *testing.T) { if err := k8sClient.Get(ctx, types.NamespacedName{Name: "mysql-lion-" + dc + "-config", Namespace: ns}, &cm); err != nil { t.Fatalf("ConfigMap for %s not created: %v", dc, err) } - if len(cm.OwnerReferences) == 0 || cm.OwnerReferences[0].Name != "lion" { - t.Errorf("ConfigMap %s owner refs = %v", dc, cm.OwnerReferences) + if !metav1.IsControlledBy(&cm, fg) { + t.Errorf("ConfigMap %s is not controlled by %s/%s: owner refs = %v", dc, fg.Namespace, fg.Name, cm.OwnerReferences) } } @@ -410,6 +416,18 @@ func TestEnvtest_ReconcilerCreatesResources(t *testing.T) { if len(svc.OwnerReferences) == 0 { t.Errorf("Service %s should have owner reference", svcName) } + if strings.HasSuffix(svcName, "-internal") { + if svc.Spec.Type != corev1.ServiceTypeClusterIP || svc.Spec.ExternalTrafficPolicy != "" || + svc.Spec.LoadBalancerIP != "" || len(svc.Spec.LoadBalancerSourceRanges) != 0 || + svc.Spec.LoadBalancerClass != nil || len(svc.Status.LoadBalancer.Ingress) != 0 { + t.Errorf("administrative Service %s is externally exposed: spec=%+v status=%+v", svcName, svc.Spec, svc.Status.LoadBalancer) + } + for _, port := range svc.Spec.Ports { + if port.NodePort != 0 { + t.Errorf("administrative Service %s port %s has nodePort %d", svcName, port.Name, port.NodePort) + } + } + } } // Verify PVCs @@ -482,11 +500,26 @@ func TestEnvtest_ReconcilerDefersExistingDeploymentSpecChange(t *testing.T) { t.Fatalf("initial reconcile failed: %v", err) } - var before appsv1.Deployment - if err := k8sClient.Get(ctx, types.NamespacedName{Name: "mysql-lion-dc1", Namespace: ns}, &before); err != nil { - t.Fatalf("deployment not found after initial reconcile: %v", err) + mysqlImage := func(deploy *appsv1.Deployment) (string, bool) { + for _, container := range deploy.Spec.Template.Spec.Containers { + if container.Name == "mysql" { + return container.Image, true + } + } + return "", false + } + initialImages := make(map[string]string, 2) + for _, site := range []string{"dc1", "dc2"} { + var before appsv1.Deployment + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "mysql-lion-" + site, Namespace: ns}, &before); err != nil { + t.Fatalf("deployment %s not found after initial reconcile: %v", site, err) + } + image, ok := mysqlImage(&before) + if !ok { + t.Fatalf("deployment %s has no named mysql container", site) + } + initialImages[site] = image } - initialImage := before.Spec.Template.Spec.Containers[0].Image // Change the image. var fetched v1alpha1.MysqlFailoverGroup @@ -504,15 +537,21 @@ func TestEnvtest_ReconcilerDefersExistingDeploymentSpecChange(t *testing.T) { t.Fatalf("re-reconcile after spec change failed: %v", err) } - // Verify the existing Deployment was left untouched. - var deploy appsv1.Deployment - if err := k8sClient.Get(ctx, types.NamespacedName{ - Name: "mysql-lion-dc1", Namespace: ns, - }, &deploy); err != nil { - t.Fatalf("deployment not found: %v", err) - } - if got := deploy.Spec.Template.Spec.Containers[0].Image; got != initialImage { - t.Errorf("existing deployment was patched outside ordered update: image %s -> %s", initialImage, got) + // Verify every existing Deployment's named MySQL container was left untouched. + for _, site := range []string{"dc1", "dc2"} { + var deploy appsv1.Deployment + if err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "mysql-lion-" + site, Namespace: ns, + }, &deploy); err != nil { + t.Fatalf("deployment %s not found: %v", site, err) + } + got, ok := mysqlImage(&deploy) + if !ok { + t.Fatalf("deployment %s has no named mysql container", site) + } + if got != initialImages[site] { + t.Errorf("existing deployment %s was patched outside ordered update: image %s -> %s", site, initialImages[site], got) + } } } From b7b292d578d40fff1596c37bfc240038c82834f1 Mon Sep 17 00:00:00 2001 From: Colin Mollenhour Date: Sat, 18 Jul 2026 06:05:00 +0000 Subject: [PATCH 7/7] Fix live-run findings from reader scenario validation All four new reader scenarios (41-44) plus the smoke profile were run against a fresh 3-site k3d playground. Two fixes from that run: - s02 follower-convergence verify: tolerate a transiently empty status.activeSite after PlannedFailoverPhaseSucceeded (the snapshot can publish one cycle before writable confirmation lands); only a different non-empty site is a terminal failure. This flaked the first smoke batch and passed on rerun with the fix. - Progress messages printed *int64 SecondsBehindSource as a pointer address (lag=0x...); add formatLag and use it in scenarios 40-44. Live results: 44 PASS 3s, 42 PASS 1m33s, 43 PASS 13s, 41 PASS 39s, then smoke profile (01, 02x2, 42) all PASS. --- internal/playground/scenarios/reader_helpers.go | 10 ++++++++++ .../playground/scenarios/s02_planned_switchover.go | 5 ++++- internal/playground/scenarios/s40_reader_data_loss.go | 4 ++-- .../s41_reader_availability_during_failover.go | 2 +- .../playground/scenarios/s42_reader_stall_isolation.go | 2 +- .../playground/scenarios/s43_writable_reader_fence.go | 2 +- .../s44_reader_source_convergence_invariant.go | 2 +- 7 files changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/playground/scenarios/reader_helpers.go b/internal/playground/scenarios/reader_helpers.go index 3fb0779..dd71728 100644 --- a/internal/playground/scenarios/reader_helpers.go +++ b/internal/playground/scenarios/reader_helpers.go @@ -3,6 +3,7 @@ package scenarios import ( "context" "fmt" + "strconv" "strings" "time" @@ -229,3 +230,12 @@ func waitForMarkerOnSite(ctx context.Context, env *runner.Env, site, qualifiedTa } return fmt.Errorf("marker %q did not replicate to %s within %s: %v", marker, site, timeout, last) } + +// formatLag renders a nullable Seconds_Behind_Source for progress +// messages ("nil" when replication is not reporting lag). +func formatLag(lag *int64) string { + if lag == nil { + return "nil" + } + return strconv.FormatInt(*lag, 10) +} diff --git a/internal/playground/scenarios/s02_planned_switchover.go b/internal/playground/scenarios/s02_planned_switchover.go index 0a2e745..a2009d8 100644 --- a/internal/playground/scenarios/s02_planned_switchover.go +++ b/internal/playground/scenarios/s02_planned_switchover.go @@ -55,7 +55,10 @@ func verifyFollowersDirectlyFollowNewPrimary() runner.Step { if err != nil { last = err.Error() } else if mfg.Status.ActiveSite == "" { - last = "activeSite is temporarily empty while topology reconverges" + // The status snapshot can publish an empty activeSite for a + // cycle while the promoted primary's writable confirmation + // lands. Only a different non-empty site is a real failure. + last = "activeSite momentarily empty after Succeeded" } else if mfg.Status.ActiveSite != target { return fmt.Errorf("planned failover target changed after success: activeSite=%q want %q", mfg.Status.ActiveSite, target) } else { diff --git a/internal/playground/scenarios/s40_reader_data_loss.go b/internal/playground/scenarios/s40_reader_data_loss.go index 9431a42..601f518 100644 --- a/internal/playground/scenarios/s40_reader_data_loss.go +++ b/internal/playground/scenarios/s40_reader_data_loss.go @@ -212,7 +212,7 @@ func s40ReplaceReaderDataAndObserve(state *s40RunState) runner.Step { return false, "reader status missing", nil } err := assertReaderServingStatus(mfg, status, state.donorHost) - return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s lag=%v", status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SecondsBehindSource), nil + return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s lag=%v", status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, formatLag(status.SecondsBehindSource)), nil }) cancelWait() if err != nil { @@ -381,7 +381,7 @@ func s40WaitForLiveReader(ctx context.Context, env *runner.Env, state *s40RunSta rs, statusErr := client.ShowReplicaStatus(ctx) markerCount, markerErr := client.ScalarInt(ctx, "SELECT COUNT(*) FROM "+s40MarkerTable+" WHERE marker=?", state.marker) _ = client.Close() - last = fmt.Sprintf("configured=%v io=%v sql=%v source=%q lag=%v marker=%d statusErr=%v markerErr=%v", rs.Configured, rs.IORunning, rs.SQLRunning, rs.SourceHost, rs.SecondsBehindSrc, markerCount, statusErr, markerErr) + last = fmt.Sprintf("configured=%v io=%v sql=%v source=%q lag=%v marker=%d statusErr=%v markerErr=%v", rs.Configured, rs.IORunning, rs.SQLRunning, rs.SourceHost, formatLag(rs.SecondsBehindSrc), markerCount, statusErr, markerErr) if statusErr == nil && markerErr == nil && rs.Configured && rs.IORunning && rs.SQLRunning && canonicalMySQLHost(rs.SourceHost) == canonicalMySQLHost(state.donorHost) && rs.SecondsBehindSrc != nil && markerCount == 1 { return nil } diff --git a/internal/playground/scenarios/s41_reader_availability_during_failover.go b/internal/playground/scenarios/s41_reader_availability_during_failover.go index e6afb57..0b0bece 100644 --- a/internal/playground/scenarios/s41_reader_availability_during_failover.go +++ b/internal/playground/scenarios/s41_reader_availability_during_failover.go @@ -148,7 +148,7 @@ func s41KillPrimaryAndObserveReader(state *s41RunState) runner.Step { } err := assertReaderServingStatus(mfg, status, state.newHost) return err == nil, fmt.Sprintf("state=%s replicating=%v source=%q convergence=%s/%s lag=%v", - status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SourceConvergenceReason, status.SecondsBehindSource), nil + status.State, status.Replicating, status.SourceHost, status.SourceConvergenceState, status.SourceConvergenceReason, formatLag(status.SecondsBehindSource)), nil }) cancelRepoint() if err != nil { diff --git a/internal/playground/scenarios/s42_reader_stall_isolation.go b/internal/playground/scenarios/s42_reader_stall_isolation.go index 49b730c..438705b 100644 --- a/internal/playground/scenarios/s42_reader_stall_isolation.go +++ b/internal/playground/scenarios/s42_reader_stall_isolation.go @@ -262,7 +262,7 @@ func s42ClearDelayAndRecover(state *s42RunState) runner.Step { } err := assertReaderServingStatus(mfg, status, state.topo.activeHost) return err == nil, fmt.Sprintf("state=%s replicating=%v convergence=%s lag=%v", - status.State, status.Replicating, status.SourceConvergenceState, status.SecondsBehindSource), nil + status.State, status.Replicating, status.SourceConvergenceState, formatLag(status.SecondsBehindSource)), nil }) if err != nil { return err diff --git a/internal/playground/scenarios/s43_writable_reader_fence.go b/internal/playground/scenarios/s43_writable_reader_fence.go index 913601a..a092a8d 100644 --- a/internal/playground/scenarios/s43_writable_reader_fence.go +++ b/internal/playground/scenarios/s43_writable_reader_fence.go @@ -390,7 +390,7 @@ func s43RecoverErrantReader(ctx context.Context, env *runner.Env, state *s43RunS } err := assertReaderServingStatus(mfg, status, state.topo.activeHost) return err == nil, fmt.Sprintf("convergence=%s/%s replicating=%v lag=%v", - status.SourceConvergenceState, status.SourceConvergenceReason, status.Replicating, status.SecondsBehindSource), nil + status.SourceConvergenceState, status.SourceConvergenceReason, status.Replicating, formatLag(status.SecondsBehindSource)), nil }) cancel() if err != nil { diff --git a/internal/playground/scenarios/s44_reader_source_convergence_invariant.go b/internal/playground/scenarios/s44_reader_source_convergence_invariant.go index b5b3424..78a4ef4 100644 --- a/internal/playground/scenarios/s44_reader_source_convergence_invariant.go +++ b/internal/playground/scenarios/s44_reader_source_convergence_invariant.go @@ -149,7 +149,7 @@ func s44VerifyDirectSourceRestored(state *s44RunState) runner.Step { } err := assertReaderServingStatus(mfg, status, state.topo.activeHost) return err == nil, fmt.Sprintf("convergence=%s/%s source=%q lag=%v", - status.SourceConvergenceState, status.SourceConvergenceReason, status.SourceHost, status.SecondsBehindSource), nil + status.SourceConvergenceState, status.SourceConvergenceReason, status.SourceHost, formatLag(status.SecondsBehindSource)), nil }) cancel() if err != nil {