mvcc: add watch failpoints for robustness testing

server/storage/mvcc/watchable_store.go

@@ -283,6 +283,7 @@ func (s *watchableStore) syncVictimsLoop() {
 
 // moveVictims tries to update watches with already pending event data
 func (s *watchableStore) moveVictims() (moved int) {
+	// gofail: var beforeMoveVictims struct{}
 	s.mu.Lock()
 	victims := s.victims
 	s.victims = nil
@@ -344,6 +345,7 @@ func (s *watchableStore) moveVictims() (moved int) {
 //  3. use minimum revision to get all key-value pairs and send those events to watchers
 //  4. remove synced watchers in set from unsynced group and move to synced group
 func (s *watchableStore) syncWatchers() int {
+	// gofail: var beforeSyncWatchers struct{}
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -513,6 +515,7 @@ func (s *watchableStore) progressAll(watchers map[WatchID]*watcher) bool {
 }
 
 func (s *watchableStore) progressIfSync(watchers map[WatchID]*watcher, responseWatchID WatchID) bool {
+	// gofail: var beforeProgressIfSync struct{}
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 

tests/robustness/failpoint/failpoint.go

@@ -53,6 +53,9 @@ var allFailpoints = []Failpoint{
 	RaftAfterSaveSleep,
 	ApplyBeforeOpenSnapshot,
 	SleepBeforeSendWatchResponse,
+	SleepBeforeSyncWatchers,
+	SleepBeforeMoveVictims,
+	SleepBeforeProgressIfSync,
 }
 
 func PickRandom(clus *e2e.EtcdProcessCluster, profile traffic.Profile) (Failpoint, error) {

tests/robustness/failpoint/gofail.go

@@ -62,6 +62,9 @@ var (
 	RaftBeforeSaveSleep                       Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second}
 	RaftAfterSaveSleep                        Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second}
 	SleepBeforeSendWatchResponse              Failpoint = gofailSleepAndDeactivate{"beforeSendWatchResponse", time.Second}
+	SleepBeforeSyncWatchers                   Failpoint = gofailSleepAndDeactivate{"beforeSyncWatchers", time.Second}
+	SleepBeforeMoveVictims                    Failpoint = gofailSleepAndDeactivate{"beforeMoveVictims", time.Second}
+	SleepBeforeProgressIfSync                 Failpoint = gofailSleepAndDeactivate{"beforeProgressIfSync", time.Second}
 )
 
 type goPanicFailpoint struct {