UpgradeTxClusters: deploy cluster-downgrade upgrade entry before load

src/FSLibrary/MissionUpgradeTxClusters.fs

@@ -65,21 +65,33 @@ let upgradeTxClusters (context: MissionContext) =
             let peer0 = formation.NetworkCfg.GetPeer coreSet 0
             peer0.WaitForMaxDependentTxClusters 8
 
-            // Don't use peer 0. This will let us run soroban_invoke on peer 1
-            // and then do the settings upgrade on peer 0.
+            // Don't use peer 0. This will let us run soroban_invoke on peer 1 and
+            // then arm the settings upgrade on peer 0 while the load is running.
             let peer1 = formation.NetworkCfg.GetPeer coreSet 1
             LogInfo "Loadgen: %s" (peer1.GenerateLoad context.SetupSorobanInvoke)
             peer1.WaitForLoadGenComplete context.SetupSorobanInvoke
-            // 1500 txs at a rate of 15 txs/sec should take ~100 seconds, so we'll have enough time to do both settings upgrades
+
+            // Deploy (but do not yet arm) the entry that lowers
+            // ledgerMaxDependentTxClusters to 4, while the network is still quiet --
+            // i.e. before starting the long-running soroban load. Deploying the
+            // upgrade-set entry takes a single Soroban transaction, and once loadgen
+            // saturates the (instruction-limited) Soroban lane that transaction gets
+            // starved and never makes it into a ledger, so the entry would never be
+            // written and the upgrade could never be armed or applied.
+            let clusterDowngradeKey =
+                formation.DeployUpgradeEntries
+                    [ coreSet ]
+                    { LoadGen.GetDefault() with
+                          mode = CreateSorobanUpgrade
+                          ledgerMaxDependentTxClusters = Some(4)
+                          minSorobanPercentSuccess = Some 100 }
+
+            // 1500 txs at a rate of 15 txs/sec should take ~100 seconds, giving us
+            // time to arm the cluster downgrade while the load is running.
             LogInfo "Loadgen: %s" (peer1.GenerateLoad context.GenerateSorobanInvokeLoad)
 
-            formation.DeployUpgradeEntriesAndArm
-                [ coreSet ]
-                { LoadGen.GetDefault() with
-                      mode = CreateSorobanUpgrade
-                      ledgerMaxDependentTxClusters = Some(4)
-                      minSorobanPercentSuccess = Some 100 }
-                (System.DateTime.UtcNow.AddSeconds(20.0))
+            // Arm the already-deployed downgrade while the load is in flight.
+            formation.ArmUpgradeEntries [ coreSet ] clusterDowngradeKey (System.DateTime.UtcNow.AddSeconds(20.0))
 
             peer0.WaitForMaxDependentTxClusters 4
 

src/FSLibrary/StellarStatefulSets.fs

@@ -362,43 +362,50 @@ type StellarFormation with
 
         self.RunLoadgen coreSet loadgen
 
-    member self.DeployUpgradeEntriesAndArm
-        (coreSetList: CoreSet list)
-        (loadGen: LoadGen)
-        (upgradeTime: System.DateTime)
-        =
+    // Deploy upgrade-set entries by running a CreateSorobanUpgrade loadgen on node 0
+    // of the first core set, and return the resulting ConfigUpgradeSetKey so the
+    // caller can arm it later. Keeping deploy separate from arm lets a caller create
+    // the upgrade-set entry while the network is quiet: deploying it takes a single
+    // Soroban transaction, and once concurrent load saturates the
+    // (instruction-limited) Soroban lane that transaction can be starved and never
+    // make it into a ledger, leaving the entry unwritten.
+    member self.DeployUpgradeEntries (coreSetList: CoreSet list) (loadGen: LoadGen) : string =
         let peer = self.NetworkCfg.GetPeer coreSetList.[0] 0
         let resStr = peer.GenerateLoad loadGen
 
-        let contractKey = Loadgen.Parse(resStr).ConfigUpgradeSetKey
+        let configUpgradeSetKey = Loadgen.Parse(resStr).ConfigUpgradeSetKey
 
         LogInfo "Loadgen: %s" resStr
         peer.WaitForLoadGenComplete loadGen
+        configUpgradeSetKey
 
-        // Arm upgrades on each peer in the core set
+    // Arm a previously-deployed upgrade-set (identified by configUpgradeSetKey) on each peer
+    // in the given core sets, to take effect at upgradeTime.
+    member self.ArmUpgradeEntries
+        (coreSetList: CoreSet list)
+        (configUpgradeSetKey: string)
+        (upgradeTime: System.DateTime)
+        =
         self.NetworkCfg.EachPeerInSets
             (List.toArray coreSetList)
-            (fun peer -> peer.UpgradeNetworkSetting contractKey upgradeTime)
+            (fun peer -> peer.UpgradeNetworkSetting configUpgradeSetKey upgradeTime)
+
+    member self.DeployUpgradeEntriesAndArm
+        (coreSetList: CoreSet list)
+        (loadGen: LoadGen)
+        (upgradeTime: System.DateTime)
+        =
+        let configUpgradeSetKey = self.DeployUpgradeEntries coreSetList loadGen
+        self.ArmUpgradeEntries coreSetList configUpgradeSetKey upgradeTime
 
     member self.DeployUpgradeEntriesAndArmAfter
         (coreSetList: CoreSet list)
         (loadGen: LoadGen)
         (delay: System.TimeSpan)
         =
-        let peer = self.NetworkCfg.GetPeer coreSetList.[0] 0
-        let resStr = peer.GenerateLoad loadGen
-
-        let contractKey = Loadgen.Parse(resStr).ConfigUpgradeSetKey
-
-        LogInfo "Loadgen: %s" resStr
-        peer.WaitForLoadGenComplete loadGen
-
+        let configUpgradeSetKey = self.DeployUpgradeEntries coreSetList loadGen
         let upgradeTime = System.DateTime.UtcNow.Add(delay)
-
-        // Arm upgrades on each peer in the core set
-        self.NetworkCfg.EachPeerInSets
-            (List.toArray coreSetList)
-            (fun peer -> peer.UpgradeNetworkSetting contractKey upgradeTime)
+        self.ArmUpgradeEntries coreSetList configUpgradeSetKey upgradeTime
 
     member self.clearMetrics(coreSets: CoreSet list) =
         self.NetworkCfg.EachPeerInSets(coreSets |> List.toArray) (fun peer -> peer.ClearMetrics())