diff --git a/driver/checking/block_height.go b/driver/checking/block_height.go index 79010cd2..bd8d82aa 100644 --- a/driver/checking/block_height.go +++ b/driver/checking/block_height.go @@ -23,6 +23,7 @@ import ( "maps" "strconv" "strings" + "sync" "github.com/0xsoniclabs/norma/driver" "github.com/0xsoniclabs/norma/driver/monitoring" @@ -64,7 +65,20 @@ func (c *blockHeightChecker) Configure(config CheckerConfig) Checker { func (c *blockHeightChecker) Check(ctx context.Context) error { nodes := c.net.GetActiveNodes() slog.Info("checking block heights for nodes", "count", len(nodes)) + + // Read all heights concurrently so the samples are taken at nearly the + // same instant; sequential reads let the chain advance between nodes and + // falsely flag early-read nodes as lagging. heights := make([]int64, len(nodes)) + errs := make([]error, len(nodes)) + var wg sync.WaitGroup + for i, n := range nodes { + wg.Go(func() { + heights[i], errs[i] = getBlockHeight(ctx, n) + }) + } + wg.Wait() + maxHeight := int64(0) expectedFailures := make(map[string]struct{}) for i, n := range nodes { @@ -72,20 +86,15 @@ func (c *blockHeightChecker) Check(ctx context.Context) error { expectedFailures[n.GetLabel()] = struct{}{} } - height, err := getBlockHeight(ctx, n) - if err != nil { - return fmt.Errorf("failed to get block height of node %s; %v", n.GetLabel(), err) - } - if height == 1 { - return fmt.Errorf("node %s reports it is at block 1 (only genesis is applied)", n.GetLabel()) + if errs[i] != nil { + return fmt.Errorf("failed to get block height of node %s; %v", n.GetLabel(), errs[i]) } - if height < 1 { - return fmt.Errorf("node %s reports it is at invalid block %d", n.GetLabel(), height) + if heights[i] < 1 { + return fmt.Errorf("node %s reports it is at invalid block %d", n.GetLabel(), heights[i]) } - if maxHeight < height { - maxHeight = height + if maxHeight < heights[i] { + maxHeight = heights[i] } - heights[i] = height } gotFailures := make(map[string]struct{}) diff --git a/driver/checking/block_height_test.go b/driver/checking/block_height_test.go index e735e438..e5305de5 100644 --- a/driver/checking/block_height_test.go +++ b/driver/checking/block_height_test.go @@ -117,26 +117,26 @@ func TestBlockHeightCheckerInvalid_WithSlack(t *testing.T) { func TestBlockHeight_ExpectedFailingNode(t *testing.T) { ctrl := gomock.NewController(t) - rpc := rpc.NewMockClient(ctrl) - rpc.EXPECT().Close().Times(2) + rpc1 := rpc.NewMockClient(ctrl) + rpc2 := rpc.NewMockClient(ctrl) node1 := driver.NewMockNode(ctrl) node1.EXPECT().IsExpectedFailure().AnyTimes().Return(false) - node1.EXPECT().DialRpc(gomock.Any()).Return(rpc, nil) + node1.EXPECT().DialRpc(gomock.Any()).Return(rpc1, nil) node1.EXPECT().GetLabel().AnyTimes().Return("node1") node2 := driver.NewMockNode(ctrl) node2.EXPECT().IsExpectedFailure().AnyTimes().Return(true) - node2.EXPECT().DialRpc(gomock.Any()).Return(rpc, nil) + node2.EXPECT().DialRpc(gomock.Any()).Return(rpc2, nil) node2.EXPECT().GetLabel().AnyTimes().Return("node2") net := driver.NewMockNetwork(ctrl) net.EXPECT().GetActiveNodes().MinTimes(1).Return([]driver.Node{node1, node2}) - gomock.InOrder( - rpc.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000"), - rpc.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "10"), // block is late - ) + rpc1.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000") + rpc1.EXPECT().Close() + rpc2.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "10") // block is late + rpc2.EXPECT().Close() c := blockHeightChecker{net: net} if err := c.Check(t.Context()); err != nil { @@ -146,29 +146,58 @@ func TestBlockHeight_ExpectedFailingNode(t *testing.T) { func TestBlockHeight_NoFailure_When_Expected(t *testing.T) { ctrl := gomock.NewController(t) - rpc := rpc.NewMockClient(ctrl) - rpc.EXPECT().Close().Times(2) + rpc1 := rpc.NewMockClient(ctrl) + rpc2 := rpc.NewMockClient(ctrl) node1 := driver.NewMockNode(ctrl) node1.EXPECT().IsExpectedFailure().AnyTimes().Return(false) - node1.EXPECT().DialRpc(gomock.Any()).Return(rpc, nil) + node1.EXPECT().DialRpc(gomock.Any()).Return(rpc1, nil) node1.EXPECT().GetLabel().AnyTimes().Return("node1") node2 := driver.NewMockNode(ctrl) node2.EXPECT().IsExpectedFailure().AnyTimes().Return(true) - node2.EXPECT().DialRpc(gomock.Any()).Return(rpc, nil) + node2.EXPECT().DialRpc(gomock.Any()).Return(rpc2, nil) node2.EXPECT().GetLabel().AnyTimes().Return("node2") net := driver.NewMockNetwork(ctrl) net.EXPECT().GetActiveNodes().MinTimes(1).Return([]driver.Node{node1, node2}) - gomock.InOrder( - rpc.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000"), - rpc.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000"), - ) + rpc1.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000") + rpc1.EXPECT().Close() + rpc2.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1000") + rpc2.EXPECT().Close() c := blockHeightChecker{net: net} if err := c.Check(t.Context()); err == nil || !strings.Contains(err.Error(), "unexpected failure set to provide the block height") { t.Errorf("unexpected error: %v", err) } } + +func TestBlockHeight_GenesisWithinSlackDoesNotFail(t *testing.T) { + ctrl := gomock.NewController(t) + rpc1 := rpc.NewMockClient(ctrl) + rpc2 := rpc.NewMockClient(ctrl) + + node1 := driver.NewMockNode(ctrl) + node1.EXPECT().IsExpectedFailure().AnyTimes().Return(false) + node1.EXPECT().DialRpc(gomock.Any()).Return(rpc1, nil) + node1.EXPECT().GetLabel().AnyTimes().Return("node1") + + node2 := driver.NewMockNode(ctrl) + node2.EXPECT().IsExpectedFailure().AnyTimes().Return(false) + node2.EXPECT().DialRpc(gomock.Any()).Return(rpc2, nil) + node2.EXPECT().GetLabel().AnyTimes().Return("node2") + + net := driver.NewMockNetwork(ctrl) + net.EXPECT().GetActiveNodes().MinTimes(1).Return([]driver.Node{node1, node2}) + + rpc1.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "1") // genesis + rpc1.EXPECT().Close() + rpc2.EXPECT().Call(gomock.Any(), "eth_blockNumber").SetArg(0, "3") + rpc2.EXPECT().Close() + + c := blockHeightChecker{net: net, slack: defaultSlack} + if err := c.Check(t.Context()); err != nil { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/driver/checking/blocks_rolling.go b/driver/checking/blocks_rolling.go index 5846d6c5..c58739fc 100644 --- a/driver/checking/blocks_rolling.go +++ b/driver/checking/blocks_rolling.go @@ -11,19 +11,24 @@ import ( const defaultToleranceSamples int = 10 +// blockSampleInterval converts a tolerance expressed in samples into an +// observation duration. Var so tests can shorten it. +var blockSampleInterval = time.Second + func init() { RegisterNetworkCheck("blocksRolling", func(net driver.Network, monitor *monitoring.Monitor) Checker { return &blocksRollingChecker{monitor: &monitoringDataAdapter{monitor}, toleranceSamples: defaultToleranceSamples} }) } -// blocksRollingChecker is a Checker checking if all nodes keeps producing blocks. +// blocksRollingChecker verifies the network is still producing blocks by +// observing for a window and requiring at least one node to advance its +// block height during it. type blocksRollingChecker struct { monitor MonitoringData toleranceSamples int - // duration, when > 0, switches the checker into live-observation mode: - // Check blocks for that long and then verifies that at least one node - // advanced its block height during the observation window. + // duration overrides the observation window; when 0 it is derived from + // toleranceSamples. duration time.Duration } @@ -58,80 +63,36 @@ func (c *blocksRollingChecker) Check(ctx context.Context) error { return fmt.Errorf("tolerance must be > 0, got %d", c.toleranceSamples) } - // Two evaluation modes are supported: - // * duration == 0 (default): a sliding window of size 'toleranceSamples' - // is walked across the entire recorded series; the node is considered - // functional if every window shows a strict block-height increase from - // its oldest to its newest sample. - // * duration > 0 (live-observation mode): mark the current time, - // sleep for the configured duration, then require that at least one - // node produced new blocks during the observation window. This models - // the intuitive question "is the network making progress right now?" - // without depending on samples recorded prior to this check. - var observationStart monitoring.Time - if c.duration > 0 { - observationStart = monitoring.Time(time.Now().UnixNano()) - timer := time.NewTimer(c.duration) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - } + window := c.duration + if window <= 0 { + window = time.Duration(c.toleranceSamples) * blockSampleInterval + } + + // Observing forward in time distinguishes a live network from one halted + // at check time and ignores history recorded before the check. + observationStart := monitoring.Time(time.Now().UnixNano()) + timer := time.NewTimer(window) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: } - // This function iterates through all nodes in the network and verifies whether their block height increases. - // A node with a stagnant block height indicates it is not actively participating in block production. - // If no nodes are found to be producing blocks, the network is deemed non-functional. - // - // The test ensures that at least one node is generating blocks, confirming that the network is operational - // to some extent. It does not verify the functionality of every node, as that is handled by other checks. - var networkFunctional bool for _, node := range c.monitor.GetNodes() { - nodeFunctional := true series := c.monitor.GetBlockStatus(node) - last := series.GetLatest() if last == nil { - //node produced no blocks continue } - - if c.duration > 0 { - // Live-observation mode: find the first sample recorded after the - // observation window started and compare its block height to the - // latest sample. Missing new samples counts as no progress. - nodeFunctional = false - for _, dp := range series.GetRange(0, last.Position+1) { - if dp.Position >= observationStart { - if dp.Value.BlockHeight < last.Value.BlockHeight { - nodeFunctional = true - } - break - } - } - } else { - // Sliding-window mode over the entire history. - items := series.GetRange(0, last.Position+1) - window := make([]monitoring.BlockStatus, c.toleranceSamples) - for i, point := range items { - window[i%c.toleranceSamples] = point.Value - if i < c.toleranceSamples-1 { - continue - } - prev := (i - c.toleranceSamples + 1) % c.toleranceSamples - if window[prev].BlockHeight >= point.Value.BlockHeight { - nodeFunctional = false - break - } - } + items := series.GetRange(observationStart, last.Position+1) + if len(items) == 0 { + continue + } + if items[0].Value.BlockHeight < last.Value.BlockHeight { + return nil } - - networkFunctional = networkFunctional || nodeFunctional } - if !networkFunctional { - return fmt.Errorf("network is down, nodes stopped producing blocks") - } - return nil + return fmt.Errorf("network is down, nodes stopped producing blocks") } diff --git a/driver/checking/blocks_rolling_test.go b/driver/checking/blocks_rolling_test.go index 2c0901da..4b549648 100644 --- a/driver/checking/blocks_rolling_test.go +++ b/driver/checking/blocks_rolling_test.go @@ -8,36 +8,26 @@ import ( "go.uber.org/mock/gomock" ) -func TestBlocksRolling_Blocks_Processed(t *testing.T) { - tests := map[string]struct { - series []uint64 - }{ - "one": { - series: []uint64{1}, - }, - "monotonic-increasing": { - series: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - }, - "monotonic-non-decreasing": { - series: []uint64{1, 2, 3, 4, 5, 5, 5, 5, 6, 7, 8, 9}, - }, - "monotonic-non-decreasing-towards-beginning": { - series: []uint64{5, 5, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13}, - }, - "monotonic-non-decreasing-towards-end": { - series: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8}, - }, - } - - for name, test := range tests { +const testObservation = time.Millisecond + +const networkDown = "network is down, nodes stopped producing blocks" + +func TestBlocksRolling_ProducingNetworkPasses(t *testing.T) { + tests := map[string][]uint64{ + "monotonic-increasing": {1, 2, 3, 4, 5}, + "flat-then-increase": {5, 5, 5, 6, 7}, + "increase-then-flat": {1, 2, 3, 3, 3}, + "single-late-increase": {4, 4, 4, 4, 5}, + } + for name, blocks := range tests { t.Run(name, func(t *testing.T) { - series := createBlockSeries(t, test.series) + series := futureSeries(t, blocks) ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5} + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} if err := c.Check(t.Context()); err != nil { t.Errorf("unexpected error: %v", err) } @@ -45,151 +35,128 @@ func TestBlocksRolling_Blocks_Processed(t *testing.T) { } } -func TestBlocksRolling_Blocks_Failure(t *testing.T) { - tests := map[string]struct { - series []uint64 - }{ - "empty": { - series: []uint64{}, - }, - "monotonic-decreasing": { - series: []uint64{10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - }, - "monotonic-non-increasing": { - series: []uint64{10, 9, 8, 7, 6, 6, 6, 6, 5, 4, 3, 2}, - }, - "non-monotonic-towards-end": { - series: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 5}, - }, - "non-monotonic-towards-beginning": { - series: []uint64{10, 1, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - }, - "monotonic-non-decreasing-long": { - series: []uint64{1, 2, 3, 4, 5, 5, 5, 5, 5, 6, 7, 8, 9}, - }, - "constant": { - series: []uint64{5, 5, 5, 5, 5, 5, 5, 5, 5, 5}, - }, - } - - for name, test := range tests { +func TestBlocksRolling_HaltedNetworkFails(t *testing.T) { + tests := map[string][]uint64{ + "empty": {}, + "single-sample": {7}, + "constant": {5, 5, 5, 5, 5}, + } + for name, blocks := range tests { t.Run(name, func(t *testing.T) { - series := createBlockSeries(t, test.series) + series := futureSeries(t, blocks) ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5} - if err := c.Check(t.Context()); err == nil || err.Error() != "network is down, nodes stopped producing blocks" { + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err == nil || err.Error() != networkDown { t.Errorf("unexpected error: %v", err) } }) } } -func TestBlocksRolling_Duration_PassesWhenBlocksAdvanceDuringObservation(t *testing.T) { - // All samples are placed in the future so they are guaranteed to fall - // inside the observation window regardless of when Check starts. - base := monitoring.Time(time.Now().UnixNano() + int64(time.Hour)) - blocks := []uint64{1, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6} - series := createBlockSeriesAt(t, base, blocks) - +func TestBlocksRolling_NoSamplesInWindowFails(t *testing.T) { + series := createBlockSeries(t, []uint64{1, 2, 3, 4, 5}) ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) - - // Sliding-window mode (duration == 0) fails on the long flat span. monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - strict := blocksRollingChecker{monitor: monitor, toleranceSamples: 5} - if err := strict.Check(t.Context()); err == nil { - t.Fatalf("expected sliding-window failure") + + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err == nil || err.Error() != networkDown { + t.Errorf("unexpected error: %v", err) } +} - // Live-observation mode: after the (short) wait, the first in-range - // sample has block height 1 and the latest 6, so the network is - // considered healthy despite the internal stall. +func TestBlocksRolling_IgnoresPastProduction(t *testing.T) { + series := mixedSeries(t, []uint64{1, 2, 3}, []uint64{3, 3, 3}) + ctrl := gomock.NewController(t) + monitor := NewMockMonitoringData(ctrl) monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - windowed := strict.Configure(CheckerConfig{"duration": int64(time.Millisecond)}) - if err := windowed.Check(t.Context()); err != nil { + + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err == nil || err.Error() != networkDown { t.Errorf("unexpected error: %v", err) } } -func TestBlocksRolling_Duration_FailsWhenBlocksStallDuringObservation(t *testing.T) { - // All samples are constant during the observation window: first-in-range - // block height equals the latest, so the check must fail. - base := monitoring.Time(time.Now().UnixNano() + int64(time.Hour)) - blocks := []uint64{5, 5, 5, 5, 5} - series := createBlockSeriesAt(t, base, blocks) - +func TestBlocksRolling_DetectsProductionAfterPastStall(t *testing.T) { + series := mixedSeries(t, []uint64{5, 5, 5}, []uint64{5, 6, 7}) ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - checker := (&blocksRollingChecker{monitor: monitor, toleranceSamples: 5}). - Configure(CheckerConfig{"duration": int64(time.Millisecond)}) - if err := checker.Check(t.Context()); err == nil || err.Error() != "network is down, nodes stopped producing blocks" { + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err != nil { t.Errorf("unexpected error: %v", err) } } -func TestBlocksRolling_Duration_FailsWhenNoSamplesAppearDuringObservation(t *testing.T) { - // All samples sit strictly in the past relative to the observation - // window; no in-range sample exists, so progress cannot be confirmed. - blocks := []uint64{1, 2, 3, 4, 5} - series := createBlockSeries(t, blocks) +func TestBlocksRolling_PassesWhenAnyNodeProduces(t *testing.T) { + ctrl := gomock.NewController(t) + monitor := NewMockMonitoringData(ctrl) + monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A", "B"}) + monitor.EXPECT().GetBlockStatus(monitoring.Node("A")).Return(futureSeries(t, []uint64{5, 5, 5})) + monitor.EXPECT().GetBlockStatus(monitoring.Node("B")).Return(futureSeries(t, []uint64{1, 2, 3})) + + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err != nil { + t.Errorf("unexpected error: %v", err) + } +} +func TestBlocksRolling_FailsWhenAllNodesHalted(t *testing.T) { ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) - monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) - monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) + monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A", "B"}) + monitor.EXPECT().GetBlockStatus(monitoring.Node("A")).Return(futureSeries(t, []uint64{5, 5, 5})) + monitor.EXPECT().GetBlockStatus(monitoring.Node("B")).Return(futureSeries(t, []uint64{9, 9, 9})) - checker := (&blocksRollingChecker{monitor: monitor, toleranceSamples: 5}). - Configure(CheckerConfig{"duration": int64(time.Millisecond)}) - if err := checker.Check(t.Context()); err == nil || err.Error() != "network is down, nodes stopped producing blocks" { + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 5, duration: testObservation} + if err := c.Check(t.Context()); err == nil || err.Error() != networkDown { t.Errorf("unexpected error: %v", err) } } -func TestBlocksRolling_Configure(t *testing.T) { - series := createBlockSeries(t, []uint64{1, 1, 1, 1, 1, 2, 3, 4, 5, 6}) +func TestBlocksRolling_DerivesWindowFromTolerance(t *testing.T) { + prev := blockSampleInterval + blockSampleInterval = time.Millisecond + defer func() { blockSampleInterval = prev }() + + series := futureSeries(t, []uint64{1, 2, 3}) ctrl := gomock.NewController(t) monitor := NewMockMonitoringData(ctrl) - monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}).Times(4) - monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series).Times(4) + monitor.EXPECT().GetNodes().Return([]monitoring.Node{"A"}) + monitor.EXPECT().GetBlockStatus(gomock.Any()).Return(series) - // original returns error because it sees 1, 1, 1, 1, 1 - original := blocksRollingChecker{monitor: monitor, toleranceSamples: 5} - if err := original.Check(t.Context()); err == nil || err.Error() != "network is down, nodes stopped producing blocks" { - t.Errorf("not caught: network is down; %v", err) + c := blocksRollingChecker{monitor: monitor, toleranceSamples: 3} + if err := c.Check(t.Context()); err != nil { + t.Errorf("unexpected error: %v", err) } +} + +func TestBlocksRolling_Configure(t *testing.T) { + orig := &blocksRollingChecker{toleranceSamples: 5, duration: 2 * time.Second} - // emptyOriginal has the same behavior as original - emptyOriginal := original.Configure(CheckerConfig{}) - if err := emptyOriginal.Check(t.Context()); err == nil || err.Error() != "network is down, nodes stopped producing blocks" { - t.Errorf("not caught: network is down; %v", err) + if got := orig.Configure(nil); got != orig { + t.Errorf("nil config should return the original checker") } - // success will pass because it sees the entire series 1->6 - success := original.Configure(CheckerConfig{"tolerance": 10}) - if err := success.Check(t.Context()); err != nil { - t.Errorf("unexpected error: %v", err) + empty := orig.Configure(CheckerConfig{}).(*blocksRollingChecker) + if empty.toleranceSamples != 5 || empty.duration != 2*time.Second { + t.Errorf("empty config should copy original values, got %+v", empty) } - // emptySuccess has the same behavior as success - emptySuccess := success.Configure(CheckerConfig{}) - if err := emptySuccess.Check(t.Context()); err != nil { - t.Errorf("unexpected error: %v", err) + set := orig.Configure(CheckerConfig{"tolerance": 7, "duration": int64(time.Millisecond)}).(*blocksRollingChecker) + if set.toleranceSamples != 7 || set.duration != time.Millisecond { + t.Errorf("config values not applied, got %+v", set) } } -// TestBlocksRolling_ZeroTolerance_ReturnsErrorInsteadOfPanic verifies that -// configuring tolerance=0 does not cause a divide-by-zero panic in the -// sliding-window path; the checker must fail cleanly with a descriptive -// error instead. func TestBlocksRolling_ZeroTolerance_ReturnsErrorInsteadOfPanic(t *testing.T) { checker := &blocksRollingChecker{toleranceSamples: 0} err := checker.Check(t.Context()) @@ -206,6 +173,12 @@ func createBlockSeries(t *testing.T, blocks []uint64) monitoring.Series[monitori return createBlockSeriesAt(t, 0, blocks) } +func futureSeries(t *testing.T, blocks []uint64) monitoring.Series[monitoring.Time, monitoring.BlockStatus] { + t.Helper() + base := monitoring.Time(time.Now().UnixNano() + int64(time.Hour)) + return createBlockSeriesAt(t, base, blocks) +} + func createBlockSeriesAt(t *testing.T, base monitoring.Time, blocks []uint64) monitoring.Series[monitoring.Time, monitoring.BlockStatus] { t.Helper() @@ -218,3 +191,20 @@ func createBlockSeriesAt(t *testing.T, base monitoring.Time, blocks []uint64) mo } return &series } + +func mixedSeries(t *testing.T, past, future []uint64) monitoring.Series[monitoring.Time, monitoring.BlockStatus] { + t.Helper() + + series := monitoring.SyncedSeries[monitoring.Time, monitoring.BlockStatus]{} + base := monitoring.Time(time.Now().UnixNano() + int64(time.Hour)) + appendAt := func(offset monitoring.Time, blocks []uint64) { + for i, block := range blocks { + if err := series.Append(offset+monitoring.Time(i), monitoring.BlockStatus{BlockHeight: block}); err != nil { + t.Fatalf("failed to append block %d: %v", block, err) + } + } + } + appendAt(0, past) + appendAt(base, future) + return &series +} diff --git a/driver/network/local/local.go b/driver/network/local/local.go index f404ceb2..aada0e2c 100644 --- a/driver/network/local/local.go +++ b/driver/network/local/local.go @@ -58,6 +58,12 @@ type LocalNetwork struct { // nodesMutex synchronizes access to the list of nodes. nodesMutex sync.Mutex + // bootstrapped is set once the first node has joined and never resets: + // the chain has history from then on, so a later (re)join must not be + // treated as a fresh bootstrap even if all nodes have been removed in + // the meantime. Guarded by nodesMutex. + bootstrapped bool + // apps maintains a list of all applications created on the network. apps []driver.Application @@ -213,12 +219,16 @@ func (n *LocalNetwork) addNodeIntoNetwork(ctx context.Context, node *node.OperaN return fmt.Errorf("failed to add peer; no existing node was reachable") } n.nodes[id] = node + n.bootstrapped = true return nil } // createNode is an internal version of CreateNode enabling the creation // of validator and non-validator nodes in the network. func (n *LocalNetwork) createNode(ctx context.Context, nodeConfig *node.OperaNodeConfig) (*node.OperaNode, error) { + n.nodesMutex.Lock() + nodeConfig.NetworkBootstrap = !n.bootstrapped + n.nodesMutex.Unlock() node, err := node.StartOperaDockerNode(ctx, n.docker, n.network, nodeConfig) if err != nil { return nil, fmt.Errorf("failed to start opera docker; %v", err) diff --git a/driver/node/opera.go b/driver/node/opera.go index f818bee9..5e5d1f8c 100644 --- a/driver/node/opera.go +++ b/driver/node/opera.go @@ -103,6 +103,9 @@ type OperaNodeConfig struct { GenesisJsonPath *string // ExtraArguments are additional command line arguments to pass to the node. ExtraArguments string + // NetworkBootstrap is true if this node starts a brand-new network, i.e. + // there is no running node it could sync with. + NetworkBootstrap bool } // imageEnsureState stores the completion signal and final error for one @@ -219,11 +222,16 @@ func StartOperaDockerNode( genesisJSONPath = *config.GenesisJsonPath } + networkBootstrap := "0" + if config.NetworkBootstrap { + networkBootstrap = "1" + } envs := map[string]string{ - "VALIDATOR_ID": validatorId, - "VALIDATORS_COUNT": fmt.Sprintf("%d", config.NetworkConfig.Validators.GetNumValidators()), - "NETWORK_LATENCY": fmt.Sprintf("%v", config.NetworkConfig.RoundTripTime/2), - "EXTRA_ARGUMENTS": config.ExtraArguments, + "VALIDATOR_ID": validatorId, + "VALIDATORS_COUNT": fmt.Sprintf("%d", config.NetworkConfig.Validators.GetNumValidators()), + "NETWORK_LATENCY": fmt.Sprintf("%v", config.NetworkConfig.RoundTripTime/2), + "EXTRA_ARGUMENTS": config.ExtraArguments, + "NETWORK_BOOTSTRAP": networkBootstrap, } const dataDir = "/datadir" diff --git a/driver/norma/run.go b/driver/norma/run.go index 02e12f02..fd385f4f 100644 --- a/driver/norma/run.go +++ b/driver/norma/run.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync" "time" @@ -365,7 +366,9 @@ func dumpNodeLogs(ctx context.Context, net driver.Network) { _ = reader.Close() if len(data) > 0 { - slog.Error("node log on failure", "node", node.GetLabel(), "log", string(data)) + label := node.GetLabel() + log := strings.TrimRight(string(data), "\n") + slog.Info("node log", "node", label, "log", log) } if err != nil && logCtx.Err() == nil { slog.Error("failed to read log", "node", node.GetLabel(), "error", err) diff --git a/scenarios/rules_upgrades/allegro_to_brio_mixed_versions_v215_forks.yml b/scenarios/rules_upgrades/allegro_to_brio_mixed_versions.yml similarity index 62% rename from scenarios/rules_upgrades/allegro_to_brio_mixed_versions_v215_forks.yml rename to scenarios/rules_upgrades/allegro_to_brio_mixed_versions.yml index d2d1c051..6c0ccd99 100644 --- a/scenarios/rules_upgrades/allegro_to_brio_mixed_versions_v215_forks.yml +++ b/scenarios/rules_upgrades/allegro_to_brio_mixed_versions.yml @@ -1,11 +1,11 @@ -Name: Rules Upgrade Allegro To Brio Mixed Versions v215 Forks +Name: Rules Upgrade Allegro To Brio Mixed Versions Forks Description: >- Upgrades network rules from Allegro to Brio with mixed validator versions. - A sonic:v2.1.5 validator forks after Brio activation and is treated as - expected failing behavior, while sonic:v2.1.6 and sonic:local validators - keep the network online via stake majority. Brio-specific large contract - load executes after enabling Brio. + The sonic:v2.1.6 validator forks after Brio activation and is treated as + expected failing behavior, while the sonic:local validator keeps the network + online via stake majority. Brio-specific large contract load executes after + enabling Brio. InitialNetworkRules: Upgrades: @@ -27,12 +27,6 @@ Scenario: stake: 2000000 failing: true - - startNode: validator-v215 - type: validator - imageName: "sonic:v2.1.5" - stake: 1000000 - failing: true - - runApp: baseline-counter type: counter users: 20 diff --git a/scripts/run_sonic.sh b/scripts/run_sonic.sh index aeeff835..d6bc779f 100755 --- a/scripts/run_sonic.sh +++ b/scripts/run_sonic.sh @@ -44,11 +44,15 @@ else fi # Create config.toml -# when network starts with only one genesis validator, then he will not wait to start emitting -# if there are two or more validators at genesis they have to wait 5 seconds after connecting to the network -# if another validator connects to the network during run it will wait also 5 seconds to start emitting +# A single genesis validator bootstrapping a brand-new network must emit +# immediately: there are no peers to sync with, and with a non-zero +# DoublesignProtection it would wait for a peer connection forever. +# In every other case — including that same validator rejoining an already +# running network (NETWORK_BOOTSTRAP=0) — keep the protection, otherwise the +# validator may emit before it has synced its own earlier events and fork +# itself, which permanently stops the node. echo '[Emitter.EmitIntervals]' >> config.toml -if [[ $VALIDATORS_COUNT == 1 && $VALIDATOR_ID == 1 ]] +if [[ $VALIDATORS_COUNT == 1 && $VALIDATOR_ID == 1 && "${NETWORK_BOOTSTRAP:-0}" == "1" ]] then echo DoublesignProtection = 0 >> config.toml else