From a480c4c71d7eea739d391bb87c99c1f9dae682af Mon Sep 17 00:00:00 2001 From: bhavani Date: Wed, 8 Apr 2026 04:10:39 +0000 Subject: [PATCH 1/6] Add 6 L3VNI DCI test cases (57, 58, 84, 85, 96, 97) - L3VNI_dci:57/58: DF failover with IPv4/IPv6 L3VNI traffic on DC2 MH leafs - L3VNI_dci:84: Simultaneous reboot of both BGWs in DC1 - L3VNI_dci:85: Rolling reboot of DC1 fabric (leaf1->leaf2->spine0->bgw1) - L3VNI_dci:96: Remove/Add import export RT on DC1 BGWs - L3VNI_dci:97: Remove/Add RT-REWRITE route-maps on DC1 BGWs Helper functions added to vxlan_helper.py: - remove_bgw_import_export_rt(): Remove RT import/export from BGW VRF config - add_bgw_import_export_rt(): Re-add RT import/export - remove_bgw_rt_rewrite_maps(): Remove RT-REWRITE route-maps + extcommunity-lists - add_bgw_rt_rewrite_maps(): Re-add RT-REWRITE route-maps + extcommunity-lists --- spytest/tests/csco-tmp/test_vxlan_dci.py | 631 +++++++++++++++++++++++ spytest/tests/csco-tmp/vxlan_helper.py | 230 +++++++++ 2 files changed, 861 insertions(+) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index 63a3c8ff677..2d99de3a973 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -6698,6 +6698,637 @@ def test_base_dci_multicast_traffic(self): report_result(result, tc_id, summ) + # ------------------------------------------------------------------ + # L3VNI_dci:57 / L3VNI_dci:58 – DF Failover with L3VNI traffic + # ------------------------------------------------------------------ + @pytest.mark.parametrize("ip_version", ["v4", "v6"]) + def test_base_dci_l3vni_mh_df_failover(self, ip_version): + """ + L3VNI_dci:57 (IPv4) / L3VNI_dci:58 (IPv6) - DF failover with L3VNI traffic. + + Description: + Multi-homed host on DC2 (leaf0_dc2 + leaf1_dc2 form an EVPN MH pair). + One leaf is the Designated Forwarder (DF) and the other is Non-DF. + Shutdown the DF leaf's host-facing PortChannel to trigger DF election + to the peer leaf, then verify L3VNI cross-DC traffic continues. + + Steps: + 1. Verify base setup before trigger + 2. Identify DC2 MH leaf pair and their PortChannels + 3. Start cross-DC L3VNI traffic (IPv4 or IPv6) + 4. Shutdown DF leaf's PortChannel to trigger DF failover + 5. Verify traffic continues after DF failover + 6. Restore PortChannel (no-shut) + 7. Verify traffic after recovery + 8. Check for core files and crashes + """ + tc_num = 57 if ip_version == 'v4' else 58 + ip_label = 'IPv4' if ip_version == 'v4' else 'IPv6' + tc_id = 'test_base_dci_l3vni_mh_{}_df_failover'.format(ip_version) + test_cfg['tc_id'] = tc_id + tc_cfg = vxlan_obj.get_tc_params(tc_id) + + st.banner('Testcase L3VNI_dci:{}: L3VNI Multi-homed - DF failover with {} L3VNI traffic ({})'.format( + tc_num, ip_label, tc_id)) + result = True + summ = '' + + # Step 1: Verify base setup before trigger + st.banner('Step 1: Verify base setup before DF failover') + setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) + if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): + summ += 'Base setup verification failed before DF failover\n' + report_result(False, tc_id, summ) + return + + # Step 2: Identify DC2 MH leaf pair and PortChannels + st.banner('Step 2: Identify DC2 MH leaf pair and PortChannels') + dc2_leafs = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) + if 'leaf' in n and 'dc2' in n] + if len(dc2_leafs) < 2: + dc2_leafs = [n for n in test_cfg['nodes'].get('l2l3vni', []) + if 'leaf' in n and 'dc2' in n] + if len(dc2_leafs) < 2: + pytest.skip('DC2 does not have 2 MH leafs for DF failover test') + return + + dc2_leafs = sorted(dc2_leafs)[:2] + st.log('DC2 MH leaf pair: {}'.format(dc2_leafs)) + + # Get PortChannel interfaces on each DC2 leaf + config_dict = vxlan_obj.get_cfg_dict() + pc_map = {} + for leaf in dc2_leafs: + leaf_cfg = config_dict.get(leaf, {}) + pcs = [] + if isinstance(leaf_cfg, dict) and leaf_cfg.get('port_channel'): + for pc_item in leaf_cfg['port_channel']: + pc_name = pc_item.get('name', '') + if pc_name: + pcs.append(pc_name) + if not pcs: + # Fallback: find PortChannels from interface config + int_cfg = vxlan_obj.get_config_interfaces_list(st.get_testbed_vars()) + l2vni_intfs = int_cfg.get(leaf, {}).get('l2vni_int', []) + pcs = [i for i in l2vni_intfs if i.startswith('PortChannel')] + pc_map[leaf] = pcs + st.log('{} PortChannels: {}'.format(leaf, pcs)) + + # Pick the first leaf as the DF target (in a real system we'd check + # 'show evpn es' but the framework doesn't always expose DF status; + # shutting either MH leaf triggers DF re-election to the peer) + df_leaf = dc2_leafs[0] + ndf_leaf = dc2_leafs[1] + df_pcs = pc_map.get(df_leaf, []) + if not df_pcs: + summ += 'No PortChannels found on DF leaf {}\n'.format(df_leaf) + report_result(False, tc_id, summ) + return + st.log('DF leaf: {} (PortChannels: {}), Peer leaf: {}'.format(df_leaf, df_pcs, ndf_leaf)) + + # Step 3: Verify cross-DC L3VNI traffic before trigger + traffic_types = ['l3_{}'.format(ip_version)] + st.banner('Step 3: Verify cross-DC L3VNI {} traffic before DF failover'.format(ip_label)) + if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): + summ += 'L3VNI {} cross-DC traffic failed before DF failover\n'.format(ip_label) + report_result(False, tc_id, summ) + return + + # Step 4: Shutdown DF leaf's PortChannel to trigger DF failover + st.banner('Step 4: Shutdown PortChannel on DF leaf {} to trigger DF failover'.format(df_leaf)) + try: + for pc in df_pcs: + st.log('Shutting PortChannel {} on {}'.format(pc, df_leaf)) + intf_obj.interface_shutdown(dut=df_leaf, interfaces=pc) + st.wait(10, 'Wait for DF election to move to peer leaf') + + # Step 5: Verify traffic continues after DF failover + st.banner('Step 5: Verify L3VNI {} cross-DC traffic after DF failover'.format(ip_label)) + if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): + summ += 'L3VNI {} cross-DC traffic failed after DF failover\n'.format(ip_label) + result = False + else: + st.log('L3VNI {} cross-DC traffic continues after DF failover: PASS'.format(ip_label)) + finally: + # Step 6: Restore PortChannel (no-shut) + st.banner('Step 6: Restore PortChannel on DF leaf {}'.format(df_leaf)) + for pc in df_pcs: + st.log('No-shutting PortChannel {} on {}'.format(pc, df_leaf)) + intf_obj.interface_noshutdown(dut=df_leaf, interfaces=pc) + st.wait(15, 'Wait for PortChannel and DF election recovery') + + # Step 7: Verify traffic after recovery + st.banner('Step 7: Verify L3VNI {} cross-DC traffic after PortChannel recovery'.format(ip_label)) + if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): + summ += 'L3VNI {} cross-DC traffic failed after PortChannel recovery\n'.format(ip_label) + result = False + else: + st.log('L3VNI {} cross-DC traffic after recovery: PASS'.format(ip_label)) + + # Step 8: Check for core files and crashes + st.banner('Step 8: Checking for core files and crashes') + if vxlan_obj.check_core(): + summ += 'Core files detected after DF failover test\n' + result = False + + report_result(result, tc_id, summ) + + # ------------------------------------------------------------------ + # L3VNI_dci:84 – Simultaneous reboot of both BGWs + # ------------------------------------------------------------------ + def test_base_dci_l3vni_simultaneous_bgw_reboot(self): + """ + L3VNI_dci:84 - Simultaneous reboot of both BGWs in DC1. + + Description: + Reboot both BGW spines of DC1 simultaneously and verify that + L2VNI + L3VNI traffic recovers after both BGWs come back up. + + Steps: + 1. Verify base setup before trigger + 2. Save FRR config on both BGWs + 3. Record docker counts + 4. Simultaneously reboot both BGWs (threaded) + 5. Verify docker recovery on both BGWs + 6. Verify base setup after reboot (with retries) + 7. Verify remote VTEPs are present + 8. Verify L2+L3 traffic flows + 9. Check for core files and crashes + """ + tc_id = 'test_base_dci_l3vni_simultaneous_bgw_reboot' + test_cfg['tc_id'] = tc_id + tc_cfg = vxlan_obj.get_tc_params(tc_id) + + st.banner('Testcase L3VNI_dci:84: Simultaneous reboot of both BGWs ({})'.format(tc_id)) + result = True + summ = '' + + # Get DC1 BGW nodes + dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) + if not dc1_bgws: + dc1_bgws = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) + if 'bgw' in n and 'dc1' in n] + if len(dc1_bgws) < 2: + pytest.skip('DC1 does not have 2 BGW nodes for simultaneous reboot test') + return + + dc1_bgws = sorted(dc1_bgws)[:2] + st.log('DC1 BGW nodes for simultaneous reboot: {}'.format(dc1_bgws)) + + # Step 1: Verify base setup before trigger + st.banner('Step 1: Verify base setup before simultaneous BGW reboot') + setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) + if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): + summ += 'Base setup verification failed before simultaneous BGW reboot\n' + report_result(False, tc_id, summ) + return + + # Step 2: Save FRR config on both BGWs + st.banner('Step 2: Save FRR configuration on both BGWs') + docker_counts = {} + for bgw in dc1_bgws: + st.log('Saving BGP config on {}'.format(bgw)) + vxlan_obj.config_dut(bgw, 'bgp', 'do write') + docker_counts[bgw] = basic_obj.get_and_match_docker_count(bgw) + st.log('Docker count on {} before reboot: {}'.format(bgw, docker_counts[bgw])) + + # Step 3: Simultaneously reboot both BGWs + st.banner('Step 3: Simultaneously rebooting both BGWs: {}'.format(dc1_bgws)) + reboot_threads = [] + for bgw in dc1_bgws: + t = threading.Thread(target=reboot_obj.dut_reboot, args=(bgw,)) + t.start() + reboot_threads.append(t) + for t in reboot_threads: + t.join() + st.log('Both BGWs have been rebooted') + + # Restore helper files + for bgw in dc1_bgws: + try: + restore_helper_file(bgw) + except Exception: + st.log('restore_helper_file not available or not needed for {}'.format(bgw)) + + # Step 4: Verify docker recovery on both BGWs + st.banner('Step 4: Verifying docker recovery on both BGWs') + for bgw in dc1_bgws: + st.log('Checking docker status on {}'.format(bgw)) + if not poll_wait(basic_obj.verify_docker_status, 180, bgw, 'Exited'): + summ += 'Dockers not auto recovered on {} after reboot\n'.format(bgw) + report_result(False, tc_id, summ) + return + if not poll_wait(basic_obj.get_and_match_docker_count, 180, bgw, docker_counts[bgw]): + summ += 'All dockers not up on {} after reboot\n'.format(bgw) + report_result(False, tc_id, summ) + return + st.log('Docker recovery verified on both BGWs') + + # Step 5: Verify base setup after reboot + st.banner('Step 5: Verifying base setup after simultaneous BGW reboot') + if not verify_base_setup_bgw(setup_nodes, retry=10): + summ += 'Base setup verification failed after simultaneous BGW reboot\n' + result = False + + # Step 6: Verify remote VTEPs + st.banner('Step 6: Verifying remote VTEPs are present') + vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) + if vtep_nodes: + if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): + summ += 'Remote VTEPs not fully recovered after simultaneous BGW reboot\n' + result = False + else: + st.log('All remote VTEPs are present') + + # Step 7: Verify L2+L3 traffic + if st.getenv('skip_tgen', 'false') != 'true': + st.banner('Step 7: Verifying traffic after simultaneous BGW reboot') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Traffic verification failed after simultaneous BGW reboot\n' + result = False + else: + st.log('L2+L3 traffic verified after simultaneous BGW reboot') + + # Step 8: Check for core files + st.banner('Step 8: Checking for core files and crashes') + if vxlan_obj.check_core(): + summ += 'Core files detected after simultaneous BGW reboot\n' + result = False + + report_result(result, tc_id, summ) + + # ------------------------------------------------------------------ + # L3VNI_dci:85 – Rolling reboot of DC fabric + # ------------------------------------------------------------------ + def test_base_dci_l3vni_rolling_reboot(self): + """ + L3VNI_dci:85 - Rolling reboot of DC1 fabric with L3VNI. + + Description: + Sequentially reboot DC1 nodes (leaf1 -> leaf2 -> spine0 -> spine1_bgw1) + and verify that L2VNI + L3VNI traffic recovers after the rolling reboot. + + Steps: + 1. Verify base setup before trigger + 2. Save FRR config on all target nodes + 3. Rolling reboot: reboot each node sequentially, wait for recovery + 4. Verify base setup after rolling reboot + 5. Verify remote VTEPs are present + 6. Verify L2+L3 traffic flows + 7. Check for core files and crashes + """ + tc_id = 'test_base_dci_l3vni_rolling_reboot' + test_cfg['tc_id'] = tc_id + tc_cfg = vxlan_obj.get_tc_params(tc_id) + + st.banner('Testcase L3VNI_dci:85: Rolling reboot of DC1 fabric with L3VNI ({})'.format(tc_id)) + result = True + summ = '' + + # Build ordered list of DC1 nodes for rolling reboot + # Order: leaf1_dc1 -> leaf2_dc1 -> spine0_dc1 -> spine1_bgw (first DC1 BGW) + all_nodes = test_cfg['nodes'].get('l2l3vni_bgw', []) + test_cfg['nodes'].get('spine', []) + dc1_leafs = sorted([n for n in all_nodes if 'leaf' in n and 'dc1' in n]) + dc1_spines = sorted([n for n in all_nodes if 'spine' in n and 'dc1' in n and 'bgw' not in n]) + dc1_bgws = sorted([n for n in all_nodes if 'bgw' in n and 'dc1' in n]) + + # Select nodes for rolling reboot: pick 2 leafs, 1 spine, 1 BGW + rolling_nodes = [] + if len(dc1_leafs) >= 2: + rolling_nodes.extend(dc1_leafs[1:3]) # leaf1_dc1, leaf2_dc1 (skip leaf0) + elif dc1_leafs: + rolling_nodes.extend(dc1_leafs[:1]) + if dc1_spines: + rolling_nodes.append(dc1_spines[0]) + if dc1_bgws: + rolling_nodes.append(dc1_bgws[0]) + + if not rolling_nodes: + pytest.skip('No DC1 nodes found for rolling reboot test') + return + + st.log('Rolling reboot order: {}'.format(rolling_nodes)) + + # Step 1: Verify base setup before trigger + st.banner('Step 1: Verify base setup before rolling reboot') + setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) + if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): + summ += 'Base setup verification failed before rolling reboot\n' + report_result(False, tc_id, summ) + return + + # Step 2: Save FRR config on all target nodes + st.banner('Step 2: Save FRR configuration on all rolling reboot nodes') + docker_counts = {} + for node in rolling_nodes: + st.log('Saving BGP config on {}'.format(node)) + vxlan_obj.config_dut(node, 'bgp', 'do write') + docker_counts[node] = basic_obj.get_and_match_docker_count(node) + + # Step 3: Rolling reboot - sequential reboot with recovery wait + st.banner('Step 3: Performing rolling reboot') + for idx, node in enumerate(rolling_nodes): + step_num = idx + 1 + st.banner('Step 3.{}: Rebooting {} ({}/{})'.format( + step_num, node, step_num, len(rolling_nodes))) + reboot_obj.dut_reboot(node) + + try: + restore_helper_file(node) + except Exception: + st.log('restore_helper_file not available or not needed for {}'.format(node)) + + st.log('Waiting for docker recovery on {}'.format(node)) + if not poll_wait(basic_obj.verify_docker_status, 180, node, 'Exited'): + summ += 'Dockers not recovered on {} during rolling reboot\n'.format(node) + result = False + continue + if not poll_wait(basic_obj.get_and_match_docker_count, 180, node, docker_counts[node]): + summ += 'All dockers not up on {} during rolling reboot\n'.format(node) + result = False + continue + st.log('{} recovered successfully'.format(node)) + + # Brief wait for BGP convergence before next reboot + st.wait(10, 'Wait for convergence after {} reboot'.format(node)) + + # Step 4: Verify base setup after rolling reboot + st.banner('Step 4: Verifying base setup after rolling reboot') + if not verify_base_setup_bgw(setup_nodes, retry=10): + summ += 'Base setup verification failed after rolling reboot\n' + result = False + + # Step 5: Verify remote VTEPs + st.banner('Step 5: Verifying remote VTEPs are present') + vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) + if vtep_nodes: + if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): + summ += 'Remote VTEPs not fully recovered after rolling reboot\n' + result = False + else: + st.log('All remote VTEPs are present') + + # Step 6: Verify L2+L3 traffic + if st.getenv('skip_tgen', 'false') != 'true': + st.banner('Step 6: Verifying traffic after rolling reboot') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Traffic verification failed after rolling reboot\n' + result = False + else: + st.log('L2+L3 traffic verified after rolling reboot') + + # Step 7: Check for core files + st.banner('Step 7: Checking for core files and crashes') + if vxlan_obj.check_core(): + summ += 'Core files detected after rolling reboot\n' + result = False + + report_result(result, tc_id, summ) + + # ------------------------------------------------------------------ + # L3VNI_dci:96 – Remove/Add Import Export RT + # ------------------------------------------------------------------ + def test_base_dci_l3vni_remove_add_import_export_rt(self): + """ + L3VNI_dci:96 - Remove/Add Import Export RT on all BGWs of DC1. + + Description: + Remove and then re-add the route-target import/export configuration + on all BGW spines of DC1. Verify that cross-DC L3VNI traffic recovers + after the RT is restored. + + Steps: + 1. Verify base setup before trigger + 2. Verify baseline cross-DC traffic + 3. Remove import/export RT on all DC1 BGWs + 4. Wait for traffic to be impacted + 5. Re-add import/export RT on all DC1 BGWs + 6. Verify traffic recovers + 7. Check for core files and crashes + """ + tc_id = 'test_base_dci_l3vni_remove_add_import_export_rt' + test_cfg['tc_id'] = tc_id + tc_cfg = vxlan_obj.get_tc_params(tc_id) + + st.banner('Testcase L3VNI_dci:96: Remove/Add Import Export RT ({})'.format(tc_id)) + result = True + summ = '' + + # Get DC1 BGW nodes + dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) + if not dc1_bgws: + dc1_bgws = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) + if 'bgw' in n and 'dc1' in n] + if not dc1_bgws: + pytest.skip('No DC1 BGW nodes found for import/export RT test') + return + + st.log('DC1 BGW nodes: {}'.format(dc1_bgws)) + + # Step 1: Verify base setup + st.banner('Step 1: Verify base setup before RT removal') + setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) + if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): + summ += 'Base setup verification failed before RT removal\n' + report_result(False, tc_id, summ) + return + + # Step 2: Verify baseline traffic + st.banner('Step 2: Verify baseline cross-DC L2+L3 traffic') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Baseline traffic verification failed before RT removal\n' + report_result(False, tc_id, summ) + return + + # Get config data for RT manipulation + config_dict = vxlan_obj.get_cfg_dict() + bgp_info = vxlan_obj.generate_bgp_underlay_info(dci_enabled=True) + + try: + # Step 3: Remove import/export RT on all DC1 BGWs + st.banner('Step 3: Remove import/export RT on DC1 BGWs') + for bgw in dc1_bgws: + st.log('Removing import/export RT on {}'.format(bgw)) + remove_cfg = vxlan_obj.remove_bgw_import_export_rt(bgw, config_dict, bgp_info) + if remove_cfg: + vxlan_obj.config_dut(bgw, 'bgp', remove_cfg) + else: + st.log('No RT config generated for {}, skipping'.format(bgw)) + + # Step 4: Wait for traffic impact + st.banner('Step 4: Wait for traffic to be impacted by RT removal') + st.wait(10, 'Wait for RT removal to take effect') + + # Step 5: Re-add import/export RT on all DC1 BGWs + st.banner('Step 5: Re-add import/export RT on DC1 BGWs') + for bgw in dc1_bgws: + st.log('Re-adding import/export RT on {}'.format(bgw)) + add_cfg = vxlan_obj.add_bgw_import_export_rt(bgw, config_dict, bgp_info) + if add_cfg: + vxlan_obj.config_dut(bgw, 'bgp', add_cfg) + else: + st.log('No RT config generated for {}, skipping'.format(bgw)) + + except Exception as e: + st.error('Exception during RT manipulation: {}'.format(e)) + # Attempt recovery + for bgw in dc1_bgws: + try: + add_cfg = vxlan_obj.add_bgw_import_export_rt(bgw, config_dict, bgp_info) + if add_cfg: + vxlan_obj.config_dut(bgw, 'bgp', add_cfg) + except Exception: + pass + summ += 'Exception during RT manipulation: {}\n'.format(e) + report_result(False, tc_id, summ) + return + + # Step 6: Verify traffic recovers + st.banner('Step 6: Wait for convergence and verify traffic recovery') + st.wait(15, 'Wait for BGP convergence after RT re-add') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Traffic did not recover after import/export RT remove/add\n' + result = False + else: + st.log('Traffic recovered after import/export RT remove/add: PASS') + + # Step 7: Check for core files + st.banner('Step 7: Checking for core files and crashes') + if vxlan_obj.check_core(): + summ += 'Core files detected after import/export RT remove/add\n' + result = False + + report_result(result, tc_id, summ) + + # ------------------------------------------------------------------ + # L3VNI_dci:97 – Remove/Add RT_REWRITE configs + # ------------------------------------------------------------------ + def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): + """ + L3VNI_dci:97 - Remove/Add RT_REWRITE configs on all BGWs of DC1. + + Description: + Remove and then re-add the RT-REWRITE route-maps (RT-REWRITE-WAN, + RT-REWRITE-DC) and associated extcommunity-lists on all BGW spines + of DC1. Verify that cross-DC L3VNI traffic recovers after configs + are restored. + + Steps: + 1. Verify base setup before trigger + 2. Verify baseline cross-DC traffic + 3. Remove RT-REWRITE configs on all DC1 BGWs + 4. Wait for traffic to be impacted + 5. Re-add RT-REWRITE configs on all DC1 BGWs + 6. Verify traffic recovers + 7. Check for core files and crashes + """ + tc_id = 'test_base_dci_l3vni_remove_add_rt_rewrite_configs' + test_cfg['tc_id'] = tc_id + tc_cfg = vxlan_obj.get_tc_params(tc_id) + + st.banner('Testcase L3VNI_dci:97: Remove/Add RT_REWRITE configs ({})'.format(tc_id)) + result = True + summ = '' + + # Get DC1 BGW nodes + dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) + if not dc1_bgws: + dc1_bgws = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) + if 'bgw' in n and 'dc1' in n] + if not dc1_bgws: + pytest.skip('No DC1 BGW nodes found for RT-REWRITE test') + return + + st.log('DC1 BGW nodes: {}'.format(dc1_bgws)) + + # Step 1: Verify base setup + st.banner('Step 1: Verify base setup before RT-REWRITE removal') + setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) + if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): + summ += 'Base setup verification failed before RT-REWRITE removal\n' + report_result(False, tc_id, summ) + return + + # Step 2: Verify baseline traffic + st.banner('Step 2: Verify baseline cross-DC L2+L3 traffic') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Baseline traffic verification failed before RT-REWRITE removal\n' + report_result(False, tc_id, summ) + return + + # Get config data for RT-REWRITE manipulation + config_dict = vxlan_obj.get_cfg_dict() + bgp_info = vxlan_obj.generate_bgp_underlay_info(dci_enabled=True) + dci_vip_maps = vxlan_obj.generate_dci_vip_maps() + + try: + # Step 3: Remove RT-REWRITE configs on all DC1 BGWs + st.banner('Step 3: Remove RT-REWRITE configs on DC1 BGWs') + for bgw in dc1_bgws: + st.log('Removing RT-REWRITE configs on {}'.format(bgw)) + remove_cfg = vxlan_obj.remove_bgw_rt_rewrite_maps(bgw, config_dict, bgp_info) + if remove_cfg: + vxlan_obj.config_dut(bgw, 'bgp', remove_cfg) + else: + st.log('No RT-REWRITE config generated for {}, skipping'.format(bgw)) + + # Step 4: Wait for traffic impact + st.banner('Step 4: Wait for traffic to be impacted by RT-REWRITE removal') + st.wait(10, 'Wait for RT-REWRITE removal to take effect') + + # Step 5: Re-add RT-REWRITE configs on all DC1 BGWs + st.banner('Step 5: Re-add RT-REWRITE configs on DC1 BGWs') + for bgw in dc1_bgws: + st.log('Re-adding RT-REWRITE configs on {}'.format(bgw)) + add_cfg = vxlan_obj.add_bgw_rt_rewrite_maps(bgw, config_dict, bgp_info, dci_vip_maps) + if add_cfg: + vxlan_obj.config_dut(bgw, 'bgp', add_cfg) + else: + st.log('No RT-REWRITE config generated for {}, skipping'.format(bgw)) + + except Exception as e: + st.error('Exception during RT-REWRITE manipulation: {}'.format(e)) + # Attempt recovery + for bgw in dc1_bgws: + try: + add_cfg = vxlan_obj.add_bgw_rt_rewrite_maps(bgw, config_dict, bgp_info, dci_vip_maps) + if add_cfg: + vxlan_obj.config_dut(bgw, 'bgp', add_cfg) + except Exception: + pass + summ += 'Exception during RT-REWRITE manipulation: {}\n'.format(e) + report_result(False, tc_id, summ) + return + + # Step 6: Verify traffic recovers + st.banner('Step 6: Wait for convergence and verify traffic recovery') + st.wait(15, 'Wait for BGP convergence after RT-REWRITE re-add') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], + scope='cross'): + summ += 'Traffic did not recover after RT-REWRITE remove/add\n' + result = False + else: + st.log('Traffic recovered after RT-REWRITE remove/add: PASS') + + # Step 7: Check for core files + st.banner('Step 7: Checking for core files and crashes') + if vxlan_obj.check_core(): + summ += 'Core files detected after RT-REWRITE remove/add\n' + result = False + + report_result(result, tc_id, summ) + # ============================================================================ # DCI MAC MOVE TRIGGER TEST CLASS (Solution_dci:71–96) diff --git a/spytest/tests/csco-tmp/vxlan_helper.py b/spytest/tests/csco-tmp/vxlan_helper.py index 4debb0ff668..a4cf84747a5 100644 --- a/spytest/tests/csco-tmp/vxlan_helper.py +++ b/spytest/tests/csco-tmp/vxlan_helper.py @@ -1279,6 +1279,236 @@ def delete_l3vni_bgw_frr_config(node_name, config_dict, bgp_info): return output +def remove_bgw_import_export_rt(node_name, config_dict, bgp_info): + """ + Remove route-target import/export from BGP VRF config on a BGW node. + + Generates FRR commands to remove all route-target export and import + statements from each VRF on the BGW, leaving the rest of the L3VNI + config (route-maps, extcommunity-lists, VRF-VNI bindings) intact. + + Args: + node_name: BGW node hostname (e.g. 'spine2_dc1_bgw1') + config_dict: Full config dict from get_cfg_dict() + bgp_info: Dict of node -> {router_id, as_num} + + Returns: + FRR config string with 'no route-target ...' commands + """ + params = _get_l3vni_bgw_params(node_name, config_dict, bgp_info) + if not params: + return '' + + as_num = params['as_num'] + remote_bgw_asns = params['remote_bgw_asns'] + leaf_asns = params['leaf_asns'] + output = '' + + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + leaf_vni = params['leaf_l3vnis_by_vrf'].get(vrf_id, 5000 + vrf_id) + output += 'router bgp {} vrf Vrf{}\n'.format(as_num, vrf_id) + output += 'address-family l2vpn evpn\n' + # Remove export + output += 'no route-target export {}:{}\n'.format(as_num, cross_dc_vni) + # Remove imports from remote BGWs + for remote_asn in remote_bgw_asns: + output += 'no route-target import {}:{}\n'.format(remote_asn, cross_dc_vni) + # Remove imports from local leaves + for leaf_asn in leaf_asns: + output += 'no route-target import {}:{}\n'.format(leaf_asn, leaf_vni) + output += 'exit-address-family\n' + output += 'exit\n' + + if output: + output += 'end\n' + output += 'exit\n' + return output + + +def add_bgw_import_export_rt(node_name, config_dict, bgp_info): + """ + Re-add route-target import/export to BGP VRF config on a BGW node. + + Generates FRR commands to restore all route-target export and import + statements for each VRF on the BGW. This is the reverse of + remove_bgw_import_export_rt. + + Args: + node_name: BGW node hostname (e.g. 'spine2_dc1_bgw1') + config_dict: Full config dict from get_cfg_dict() + bgp_info: Dict of node -> {router_id, as_num} + + Returns: + FRR config string with 'route-target ...' commands + """ + params = _get_l3vni_bgw_params(node_name, config_dict, bgp_info) + if not params: + return '' + + as_num = params['as_num'] + remote_bgw_asns = params['remote_bgw_asns'] + leaf_asns = params['leaf_asns'] + output = '' + + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + leaf_vni = params['leaf_l3vnis_by_vrf'].get(vrf_id, 5000 + vrf_id) + output += 'router bgp {} vrf Vrf{}\n'.format(as_num, vrf_id) + output += 'address-family l2vpn evpn\n' + # Re-add export + output += 'route-target export {}:{}\n'.format(as_num, cross_dc_vni) + # Re-add imports from remote BGWs + for remote_asn in remote_bgw_asns: + output += 'route-target import {}:{}\n'.format(remote_asn, cross_dc_vni) + # Re-add imports from local leaves + for leaf_asn in leaf_asns: + output += 'route-target import {}:{}\n'.format(leaf_asn, leaf_vni) + output += 'exit-address-family\n' + output += 'exit\n' + + if output: + output += 'end\n' + output += 'exit\n' + return output + + +def remove_bgw_rt_rewrite_maps(node_name, config_dict, bgp_info): + """ + Remove RT-REWRITE route-maps and extcommunity-lists from a BGW node. + + Generates FRR commands to remove: + - RT-REWRITE-WAN and RT-REWRITE-DC route-maps + - The 'neighbor ... route-map' application on OVERLAY and OVERLAY_WAN + - RT-WAN-* and RT-DC-* extcommunity-lists + + The BGP VRF route-target import/export and VRF-VNI bindings are left intact. + + Args: + node_name: BGW node hostname (e.g. 'spine2_dc1_bgw1') + config_dict: Full config dict from get_cfg_dict() + bgp_info: Dict of node -> {router_id, as_num} + + Returns: + FRR config string + """ + params = _get_l3vni_bgw_params(node_name, config_dict, bgp_info) + if not params: + return '' + + as_num = params['as_num'] + output = '' + + # Remove route-map application from BGP neighbors + output += 'router bgp {}\n'.format(as_num) + output += 'address-family l2vpn evpn\n' + output += 'no neighbor OVERLAY route-map RT-REWRITE-DC out\n' + output += 'no neighbor OVERLAY_WAN route-map RT-REWRITE-WAN out\n' + output += 'exit-address-family\n' + output += 'exit\n' + + # Remove route-maps + output += 'no route-map RT-REWRITE-WAN\n' + output += 'no route-map RT-REWRITE-DC\n' + + # Remove extcommunity-lists + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + output += 'no bgp extcommunity-list standard RT-WAN-{}\n'.format(cross_dc_vni) + output += 'no bgp extcommunity-list standard RT-DC-{}\n'.format(cross_dc_vni) + + output += 'end\n' + output += 'exit\n' + return output + + +def add_bgw_rt_rewrite_maps(node_name, config_dict, bgp_info, dci_vip_maps): + """ + Re-add RT-REWRITE route-maps and extcommunity-lists on a BGW node. + + Generates FRR commands to restore: + - RT-WAN-* and RT-DC-* extcommunity-lists + - RT-REWRITE-WAN route-map (WAN-side: IPv4 next-hop) + - RT-REWRITE-DC route-map (DC-side: IPv6 next-hop) + - Apply route-maps to OVERLAY and OVERLAY_WAN neighbors + + This is the reverse of remove_bgw_rt_rewrite_maps. + + Args: + node_name: BGW node hostname (e.g. 'spine2_dc1_bgw1') + config_dict: Full config dict from get_cfg_dict() + bgp_info: Dict of node -> {router_id, as_num} + dci_vip_maps: Tuple from generate_dci_vip_maps(): + (loopback_ipv6_dc_vip, loopback_ipv4_wan_vip, ...) + + Returns: + FRR config string + """ + params = _get_l3vni_bgw_params(node_name, config_dict, bgp_info) + if not params: + return '' + + loopback_ipv6_dc_vip, loopback_ipv4_wan_vip = dci_vip_maps[0], dci_vip_maps[1] + as_num = params['as_num'] + dc_vip_ipv6 = loopback_ipv6_dc_vip.get(node_name, '') + wan_vip_ipv4 = loopback_ipv4_wan_vip.get(node_name, '') + leaf_asns = params['leaf_asns'] + remote_bgw_asns = params['remote_bgw_asns'] + + output = '' + + # 1) Extcommunity-lists + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + leaf_vni = params['leaf_l3vnis_by_vrf'].get(vrf_id, 5000 + vrf_id) + seq = 5 + for leaf_asn in leaf_asns: + output += 'bgp extcommunity-list standard RT-WAN-{} seq {} permit rt {}:{}\n'.format( + cross_dc_vni, seq, leaf_asn, leaf_vni) + seq += 5 + seq = 5 + for remote_asn in remote_bgw_asns: + output += 'bgp extcommunity-list standard RT-DC-{} seq {} permit rt {}:{}\n'.format( + cross_dc_vni, seq, remote_asn, cross_dc_vni) + seq += 5 + + # 2) RT-REWRITE-WAN route-map + permit_seq = 10 + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + output += 'route-map RT-REWRITE-WAN permit {}\n'.format(permit_seq) + output += ' match extcommunity RT-WAN-{}\n'.format(cross_dc_vni) + output += ' match evpn route-type prefix\n' + output += ' set evpn vni {}\n'.format(cross_dc_vni) + output += ' set evpn rmac local\n' + output += ' set extcommunity rt {}:{}\n'.format(as_num, cross_dc_vni) + output += ' set ip next-hop {}\n'.format(wan_vip_ipv4) + output += 'exit\n' + permit_seq += 5 + output += 'route-map RT-REWRITE-WAN permit {}\n'.format(permit_seq) + output += 'exit\n' + + # 3) RT-REWRITE-DC route-map + permit_seq = 10 + for vrf_id, cross_dc_vni in params['vrf_l3vni_pairs']: + output += 'route-map RT-REWRITE-DC permit {}\n'.format(permit_seq) + output += ' match extcommunity RT-DC-{}\n'.format(cross_dc_vni) + output += ' match evpn route-type prefix\n' + output += ' set evpn vni {}\n'.format(cross_dc_vni) + output += ' set evpn rmac local\n' + output += ' set extcommunity rt {}:{}\n'.format(as_num, cross_dc_vni) + output += ' set ipv6 next-hop global {}\n'.format(dc_vip_ipv6) + output += 'exit\n' + permit_seq += 5 + output += 'route-map RT-REWRITE-DC permit {}\n'.format(permit_seq) + output += 'exit\n' + + # 4) Apply route-maps to OVERLAY and OVERLAY_WAN neighbors + output += 'router bgp {}\n'.format(as_num) + output += 'address-family l2vpn evpn\n' + output += 'neighbor OVERLAY route-map RT-REWRITE-DC out\n' + output += 'neighbor OVERLAY_WAN route-map RT-REWRITE-WAN out\n' + output += 'exit-address-family\n' + output += 'exit\n' + + return output + + def generate_l3vni_leaf_rt_config(node_name, config_dict, bgp_info): """ Generate leaf VRF route-target export/import for intra-DC EVPN and From 290621b96ca5d32601439442d0b1426c2ecb30cc Mon Sep 17 00:00:00 2001 From: bhavani Date: Thu, 9 Apr 2026 21:37:44 +0000 Subject: [PATCH 2/6] Update all 6 L3VNI DCI test cases to use continuous traffic (dci_flap_continuous) Per reviewer feedback, all test cases now use dci_flap_continuous streams: - TC57/58 (DF failover): Start continuous cross-DC traffic before shutdown, verify continuous traffic continues after DF failover (check with min_perc) - TC84 (simultaneous BGW reboot): Start continuous L2+L3 before reboot, stop after recovery (drop not measured), burst verify across+within DC - TC85 (rolling reboot): Start continuous L2+L3 before reboot, stop after recovery (drop not measured), burst verify across+within DC - TC96 (remove/add RT): Start continuous, remove/add RT, verify continuous recovers within threshold (min_perc=99.6) - TC97 (remove/add RT-REWRITE): Same pattern as TC96 Traffic pattern follows test_dci_link_trigger (InterfaceTrigger class): vxlan_obj.check_traffic(fc_streams, action='start/check/stop', ...) --- spytest/tests/csco-tmp/test_vxlan_dci.py | 589 ++++++++++++++--------- 1 file changed, 355 insertions(+), 234 deletions(-) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index 2d99de3a973..f0d5c8a40be 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -6700,6 +6700,8 @@ def test_base_dci_multicast_traffic(self): # ------------------------------------------------------------------ # L3VNI_dci:57 / L3VNI_dci:58 – DF Failover with L3VNI traffic + # Uses continuous cross-DC traffic (dci_flap_continuous) per + # InterfaceTrigger pattern. # ------------------------------------------------------------------ @pytest.mark.parametrize("ip_version", ["v4", "v6"]) def test_base_dci_l3vni_mh_df_failover(self, ip_version): @@ -6709,18 +6711,18 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): Description: Multi-homed host on DC2 (leaf0_dc2 + leaf1_dc2 form an EVPN MH pair). One leaf is the Designated Forwarder (DF) and the other is Non-DF. - Shutdown the DF leaf's host-facing PortChannel to trigger DF election - to the peer leaf, then verify L3VNI cross-DC traffic continues. + Start continuous cross-DC traffic, shutdown the DF leaf's host-facing + PortChannel to trigger DF election to the peer leaf, then verify + continuous traffic continues without significant loss. Steps: - 1. Verify base setup before trigger - 2. Identify DC2 MH leaf pair and their PortChannels - 3. Start cross-DC L3VNI traffic (IPv4 or IPv6) - 4. Shutdown DF leaf's PortChannel to trigger DF failover - 5. Verify traffic continues after DF failover - 6. Restore PortChannel (no-shut) - 7. Verify traffic after recovery - 8. Check for core files and crashes + 1. Verify base setup + 2. Pick Leaf0 and Leaf1 of DC2 (one DF, one NDF) + 3. Start continuous cross-DC traffic (dci_flap_continuous) + 4. Shutdown DF link of multi-homed host + 5. Verify DF election moves to peer leaf, continuous traffic continues + 6. Restore PortChannel (no-shut), stop continuous traffic + 7. Check for core files and crashes """ tc_num = 57 if ip_version == 'v4' else 58 ip_label = 'IPv4' if ip_version == 'v4' else 'IPv6' @@ -6732,8 +6734,12 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): tc_num, ip_label, tc_id)) result = True summ = '' + stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) + start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) + fc_streams = tgen_handles.get('dci_flap_continuous') + fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - # Step 1: Verify base setup before trigger + # Step 1: Verify base setup st.banner('Step 1: Verify base setup before DF failover') setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) if not verify_base_setup_bgw(setup_nodes, skip_checks=['vteps']): @@ -6741,8 +6747,8 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): report_result(False, tc_id, summ) return - # Step 2: Identify DC2 MH leaf pair and PortChannels - st.banner('Step 2: Identify DC2 MH leaf pair and PortChannels') + # Step 2: Pick Leaf0 and Leaf1 of DC2 (DF/NDF pair) + st.banner('Step 2: Pick Leaf0 and Leaf1 of DC2 as DF/NDF pair') dc2_leafs = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) if 'leaf' in n and 'dc2' in n] if len(dc2_leafs) < 2: @@ -6767,16 +6773,12 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): if pc_name: pcs.append(pc_name) if not pcs: - # Fallback: find PortChannels from interface config int_cfg = vxlan_obj.get_config_interfaces_list(st.get_testbed_vars()) l2vni_intfs = int_cfg.get(leaf, {}).get('l2vni_int', []) pcs = [i for i in l2vni_intfs if i.startswith('PortChannel')] pc_map[leaf] = pcs st.log('{} PortChannels: {}'.format(leaf, pcs)) - # Pick the first leaf as the DF target (in a real system we'd check - # 'show evpn es' but the framework doesn't always expose DF status; - # shutting either MH leaf triggers DF re-election to the peer) df_leaf = dc2_leafs[0] ndf_leaf = dc2_leafs[1] df_pcs = pc_map.get(df_leaf, []) @@ -6786,47 +6788,58 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): return st.log('DF leaf: {} (PortChannels: {}), Peer leaf: {}'.format(df_leaf, df_pcs, ndf_leaf)) - # Step 3: Verify cross-DC L3VNI traffic before trigger - traffic_types = ['l3_{}'.format(ip_version)] - st.banner('Step 3: Verify cross-DC L3VNI {} traffic before DF failover'.format(ip_label)) - if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): - summ += 'L3VNI {} cross-DC traffic failed before DF failover\n'.format(ip_label) + # Step 3: Start continuous cross-DC traffic + st.banner('Step 3: Start continuous {} cross-DC traffic (dci_flap_continuous)'.format(ip_label)) + if not fc_available: + summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' + report_result(False, tc_id, summ) + return + + try: + vxlan_obj.check_traffic( + fc_streams, + regenerate_traffic_items=True, + action='start', + stop_proto_wait=stop_pw, + start_proto_wait=start_pw, + ) + except Exception as err: + st.error('Continuous traffic start failed: {}'.format(err)) + summ += 'Continuous traffic start failed: {}\n'.format(err) report_result(False, tc_id, summ) return - # Step 4: Shutdown DF leaf's PortChannel to trigger DF failover - st.banner('Step 4: Shutdown PortChannel on DF leaf {} to trigger DF failover'.format(df_leaf)) try: + # Step 4: Shutdown DF link of multi-homed host + st.banner('Step 4: Shutdown PortChannel on DF leaf {} to trigger DF failover'.format(df_leaf)) for pc in df_pcs: st.log('Shutting PortChannel {} on {}'.format(pc, df_leaf)) intf_obj.interface_shutdown(dut=df_leaf, interfaces=pc) st.wait(10, 'Wait for DF election to move to peer leaf') - # Step 5: Verify traffic continues after DF failover - st.banner('Step 5: Verify L3VNI {} cross-DC traffic after DF failover'.format(ip_label)) - if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): - summ += 'L3VNI {} cross-DC traffic failed after DF failover\n'.format(ip_label) + # Step 5: Verify continuous traffic continues after DF failover + st.banner('Step 5: Verify continuous {} cross-DC traffic after DF failover'.format(ip_label)) + if not vxlan_obj.check_traffic( + fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): + summ += 'Continuous {} cross-DC traffic failed after DF failover\n'.format(ip_label) result = False else: - st.log('L3VNI {} cross-DC traffic continues after DF failover: PASS'.format(ip_label)) + st.log('Continuous {} cross-DC traffic continues after DF failover: PASS'.format(ip_label)) + finally: - # Step 6: Restore PortChannel (no-shut) - st.banner('Step 6: Restore PortChannel on DF leaf {}'.format(df_leaf)) + # Step 6: Restore PortChannel and stop continuous traffic + st.banner('Step 6: Restore PortChannel on DF leaf {} and stop continuous traffic'.format(df_leaf)) for pc in df_pcs: st.log('No-shutting PortChannel {} on {}'.format(pc, df_leaf)) intf_obj.interface_noshutdown(dut=df_leaf, interfaces=pc) + try: + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + except Exception: + pass st.wait(15, 'Wait for PortChannel and DF election recovery') - # Step 7: Verify traffic after recovery - st.banner('Step 7: Verify L3VNI {} cross-DC traffic after PortChannel recovery'.format(ip_label)) - if not verify_traffic(tgen_handles, traffic_types=traffic_types, scope='cross'): - summ += 'L3VNI {} cross-DC traffic failed after PortChannel recovery\n'.format(ip_label) - result = False - else: - st.log('L3VNI {} cross-DC traffic after recovery: PASS'.format(ip_label)) - - # Step 8: Check for core files and crashes - st.banner('Step 8: Checking for core files and crashes') + # Step 7: Check for core files and crashes + st.banner('Step 7: Checking for core files and crashes') if vxlan_obj.check_core(): summ += 'Core files detected after DF failover test\n' result = False @@ -6835,25 +6848,30 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): # ------------------------------------------------------------------ # L3VNI_dci:84 – Simultaneous reboot of both BGWs + # Uses continuous cross-DC traffic (dci_flap_continuous); drop on the + # continuous stream is NOT measured – burst verification after recovery. # ------------------------------------------------------------------ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): """ L3VNI_dci:84 - Simultaneous reboot of both BGWs in DC1. Description: - Reboot both BGW spines of DC1 simultaneously and verify that - L2VNI + L3VNI traffic recovers after both BGWs come back up. + Start continuous L2+L3 cross-DC traffic, simultaneously reboot + both BGW spines of DC1, verify docker recovery and VTEPs, stop + the continuous stream (drop not measured), then burst-verify + L2+L3 traffic across and within DC. Steps: - 1. Verify base setup before trigger - 2. Save FRR config on both BGWs - 3. Record docker counts - 4. Simultaneously reboot both BGWs (threaded) - 5. Verify docker recovery on both BGWs - 6. Verify base setup after reboot (with retries) - 7. Verify remote VTEPs are present - 8. Verify L2+L3 traffic flows - 9. Check for core files and crashes + 1. Verify base setup before reboot + 2. Save FRR configuration on both BGWs of DC1 + 3. Start continuous L2 and L3 traffic across DC + 4. Simultaneously reboot both BGWs of DC1 + 5. Verify docker recovery of both BGWs + 6. Verify all remote VTEPs are present with retries; + stop continuous traffic (not measuring drop) + 7. Verify traffic flows l2_v4, l2_v6, l3_v4, l3_v6 + across and within DC + 8. Check for core files/crashes """ tc_id = 'test_base_dci_l3vni_simultaneous_bgw_reboot' test_cfg['tc_id'] = tc_id @@ -6862,6 +6880,10 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): st.banner('Testcase L3VNI_dci:84: Simultaneous reboot of both BGWs ({})'.format(tc_id)) result = True summ = '' + stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) + start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) + fc_streams = tgen_handles.get('dci_flap_continuous') + fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) @@ -6892,64 +6914,88 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): docker_counts[bgw] = basic_obj.get_and_match_docker_count(bgw) st.log('Docker count on {} before reboot: {}'.format(bgw, docker_counts[bgw])) - # Step 3: Simultaneously reboot both BGWs - st.banner('Step 3: Simultaneously rebooting both BGWs: {}'.format(dc1_bgws)) - reboot_threads = [] - for bgw in dc1_bgws: - t = threading.Thread(target=reboot_obj.dut_reboot, args=(bgw,)) - t.start() - reboot_threads.append(t) - for t in reboot_threads: - t.join() - st.log('Both BGWs have been rebooted') - - # Restore helper files - for bgw in dc1_bgws: + # Step 3: Start continuous L2+L3 traffic across DC + st.banner('Step 3: Start continuous L2+L3 traffic across DC (dci_flap_continuous)') + if fc_available: try: - restore_helper_file(bgw) - except Exception: - st.log('restore_helper_file not available or not needed for {}'.format(bgw)) + vxlan_obj.check_traffic( + fc_streams, + regenerate_traffic_items=True, + action='start', + stop_proto_wait=stop_pw, + start_proto_wait=start_pw, + ) + except Exception as err: + st.error('Continuous traffic start failed: {}'.format(err)) + summ += 'Continuous traffic start failed: {}\n'.format(err) + result = False + else: + st.log('dci_flap_continuous streams not available; will use burst verification only') - # Step 4: Verify docker recovery on both BGWs - st.banner('Step 4: Verifying docker recovery on both BGWs') - for bgw in dc1_bgws: - st.log('Checking docker status on {}'.format(bgw)) - if not poll_wait(basic_obj.verify_docker_status, 180, bgw, 'Exited'): - summ += 'Dockers not auto recovered on {} after reboot\n'.format(bgw) - report_result(False, tc_id, summ) - return - if not poll_wait(basic_obj.get_and_match_docker_count, 180, bgw, docker_counts[bgw]): - summ += 'All dockers not up on {} after reboot\n'.format(bgw) - report_result(False, tc_id, summ) - return - st.log('Docker recovery verified on both BGWs') + # Step 4: Simultaneously reboot both BGWs + try: + st.banner('Step 4: Simultaneously rebooting both BGWs: {}'.format(dc1_bgws)) + reboot_threads = [] + for bgw in dc1_bgws: + t = threading.Thread(target=reboot_obj.dut_reboot, args=(bgw,)) + t.start() + reboot_threads.append(t) + for t in reboot_threads: + t.join() + st.log('Both BGWs have been rebooted') + + # Restore helper files + for bgw in dc1_bgws: + try: + restore_helper_file(bgw) + except Exception: + st.log('restore_helper_file not available or not needed for {}'.format(bgw)) - # Step 5: Verify base setup after reboot - st.banner('Step 5: Verifying base setup after simultaneous BGW reboot') - if not verify_base_setup_bgw(setup_nodes, retry=10): - summ += 'Base setup verification failed after simultaneous BGW reboot\n' - result = False + # Step 5: Verify docker recovery on both BGWs + st.banner('Step 5: Verifying docker recovery on both BGWs') + for bgw in dc1_bgws: + st.log('Checking docker status on {}'.format(bgw)) + if not poll_wait(basic_obj.verify_docker_status, 180, bgw, 'Exited'): + summ += 'Dockers not auto recovered on {} after reboot\n'.format(bgw) + report_result(False, tc_id, summ) + return + if not poll_wait(basic_obj.get_and_match_docker_count, 180, bgw, docker_counts[bgw]): + summ += 'All dockers not up on {} after reboot\n'.format(bgw) + report_result(False, tc_id, summ) + return + st.log('Docker recovery verified on both BGWs') - # Step 6: Verify remote VTEPs - st.banner('Step 6: Verifying remote VTEPs are present') - vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) - if vtep_nodes: - if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): - summ += 'Remote VTEPs not fully recovered after simultaneous BGW reboot\n' + # Step 6: Verify VTEPs with retries + st.banner('Step 6: Verifying remote VTEPs are present with retries') + if not verify_base_setup_bgw(setup_nodes, retry=10): + summ += 'Base setup verification failed after simultaneous BGW reboot\n' result = False - else: - st.log('All remote VTEPs are present') - # Step 7: Verify L2+L3 traffic - if st.getenv('skip_tgen', 'false') != 'true': - st.banner('Step 7: Verifying traffic after simultaneous BGW reboot') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Traffic verification failed after simultaneous BGW reboot\n' - result = False - else: - st.log('L2+L3 traffic verified after simultaneous BGW reboot') + vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) + if vtep_nodes: + if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): + summ += 'Remote VTEPs not fully recovered after simultaneous BGW reboot\n' + result = False + else: + st.log('All remote VTEPs are present') + + finally: + # Stop continuous traffic (not measuring drop on continuous stream) + if fc_available and fc_streams: + st.banner('Stopping continuous traffic (drop not measured on continuous stream)') + try: + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + except Exception: + pass + + # Step 7: Verify burst traffic l2_v4, l2_v6, l3_v4, l3_v6 across and within DC + st.banner('Step 7: Verifying burst traffic across and within DC after simultaneous BGW reboot') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6']): + summ += 'Traffic verification failed after simultaneous BGW reboot\n' + result = False + else: + st.log('L2+L3 traffic verified across and within DC after simultaneous BGW reboot') # Step 8: Check for core files st.banner('Step 8: Checking for core files and crashes') @@ -6961,23 +7007,30 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): # ------------------------------------------------------------------ # L3VNI_dci:85 – Rolling reboot of DC fabric + # Uses continuous cross-DC traffic (dci_flap_continuous); drop on the + # continuous stream is NOT measured – burst verification after recovery. # ------------------------------------------------------------------ def test_base_dci_l3vni_rolling_reboot(self): """ L3VNI_dci:85 - Rolling reboot of DC1 fabric with L3VNI. Description: - Sequentially reboot DC1 nodes (leaf1 -> leaf2 -> spine0 -> spine1_bgw1) - and verify that L2VNI + L3VNI traffic recovers after the rolling reboot. + Start continuous L2+L3 cross-DC traffic, sequentially reboot DC1 + nodes (Leaf1 -> Leaf2 -> Spine0 -> spine1_bgw1), verify docker + recovery of all rebooted nodes, stop continuous stream (drop not + measured), then burst-verify L2+L3 traffic across and within DC. Steps: - 1. Verify base setup before trigger - 2. Save FRR config on all target nodes - 3. Rolling reboot: reboot each node sequentially, wait for recovery - 4. Verify base setup after rolling reboot - 5. Verify remote VTEPs are present - 6. Verify L2+L3 traffic flows - 7. Check for core files and crashes + 1. Verify base setup before reboot + 2. Save FRR configuration on leaf1, leaf2, spine0, spine1_bgw1 of DC1 + 3. Start continuous L2 and L3 traffic across and within DC + 4. Perform rolling reboot: Leaf1 -> Leaf2 -> Spine0 -> spine1_bgw1 + 5. Verify docker recovery of all rebooted nodes + 6. Verify all remote VTEPs are present with retries; + stop continuous traffic (not measuring drop) + 7. Verify traffic flows l2_v4, l2_v6, l3_v4, l3_v6 + across and within DC + 8. Check for core files/crashes """ tc_id = 'test_base_dci_l3vni_rolling_reboot' test_cfg['tc_id'] = tc_id @@ -6986,6 +7039,10 @@ def test_base_dci_l3vni_rolling_reboot(self): st.banner('Testcase L3VNI_dci:85: Rolling reboot of DC1 fabric with L3VNI ({})'.format(tc_id)) result = True summ = '' + stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) + start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) + fc_streams = tgen_handles.get('dci_flap_continuous') + fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 # Build ordered list of DC1 nodes for rolling reboot # Order: leaf1_dc1 -> leaf2_dc1 -> spine0_dc1 -> spine1_bgw (first DC1 BGW) @@ -6994,7 +7051,6 @@ def test_base_dci_l3vni_rolling_reboot(self): dc1_spines = sorted([n for n in all_nodes if 'spine' in n and 'dc1' in n and 'bgw' not in n]) dc1_bgws = sorted([n for n in all_nodes if 'bgw' in n and 'dc1' in n]) - # Select nodes for rolling reboot: pick 2 leafs, 1 spine, 1 BGW rolling_nodes = [] if len(dc1_leafs) >= 2: rolling_nodes.extend(dc1_leafs[1:3]) # leaf1_dc1, leaf2_dc1 (skip leaf0) @@ -7027,62 +7083,87 @@ def test_base_dci_l3vni_rolling_reboot(self): vxlan_obj.config_dut(node, 'bgp', 'do write') docker_counts[node] = basic_obj.get_and_match_docker_count(node) - # Step 3: Rolling reboot - sequential reboot with recovery wait - st.banner('Step 3: Performing rolling reboot') - for idx, node in enumerate(rolling_nodes): - step_num = idx + 1 - st.banner('Step 3.{}: Rebooting {} ({}/{})'.format( - step_num, node, step_num, len(rolling_nodes))) - reboot_obj.dut_reboot(node) - + # Step 3: Start continuous L2+L3 traffic across and within DC + st.banner('Step 3: Start continuous L2+L3 traffic across and within DC (dci_flap_continuous)') + if fc_available: try: - restore_helper_file(node) - except Exception: - st.log('restore_helper_file not available or not needed for {}'.format(node)) - - st.log('Waiting for docker recovery on {}'.format(node)) - if not poll_wait(basic_obj.verify_docker_status, 180, node, 'Exited'): - summ += 'Dockers not recovered on {} during rolling reboot\n'.format(node) - result = False - continue - if not poll_wait(basic_obj.get_and_match_docker_count, 180, node, docker_counts[node]): - summ += 'All dockers not up on {} during rolling reboot\n'.format(node) + vxlan_obj.check_traffic( + fc_streams, + regenerate_traffic_items=True, + action='start', + stop_proto_wait=stop_pw, + start_proto_wait=start_pw, + ) + except Exception as err: + st.error('Continuous traffic start failed: {}'.format(err)) + summ += 'Continuous traffic start failed: {}\n'.format(err) result = False - continue - st.log('{} recovered successfully'.format(node)) + else: + st.log('dci_flap_continuous streams not available; will use burst verification only') - # Brief wait for BGP convergence before next reboot - st.wait(10, 'Wait for convergence after {} reboot'.format(node)) + # Step 4: Rolling reboot - sequential reboot with recovery wait + try: + st.banner('Step 4: Performing rolling reboot') + for idx, node in enumerate(rolling_nodes): + step_num = idx + 1 + st.banner('Step 4.{}: Rebooting {} ({}/{})'.format( + step_num, node, step_num, len(rolling_nodes))) + reboot_obj.dut_reboot(node) - # Step 4: Verify base setup after rolling reboot - st.banner('Step 4: Verifying base setup after rolling reboot') - if not verify_base_setup_bgw(setup_nodes, retry=10): - summ += 'Base setup verification failed after rolling reboot\n' - result = False + try: + restore_helper_file(node) + except Exception: + st.log('restore_helper_file not available or not needed for {}'.format(node)) - # Step 5: Verify remote VTEPs - st.banner('Step 5: Verifying remote VTEPs are present') - vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) - if vtep_nodes: - if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): - summ += 'Remote VTEPs not fully recovered after rolling reboot\n' - result = False - else: - st.log('All remote VTEPs are present') + # Step 5 (per node): Verify docker recovery + st.log('Waiting for docker recovery on {}'.format(node)) + if not poll_wait(basic_obj.verify_docker_status, 180, node, 'Exited'): + summ += 'Dockers not recovered on {} during rolling reboot\n'.format(node) + result = False + continue + if not poll_wait(basic_obj.get_and_match_docker_count, 180, node, docker_counts[node]): + summ += 'All dockers not up on {} during rolling reboot\n'.format(node) + result = False + continue + st.log('{} recovered successfully'.format(node)) - # Step 6: Verify L2+L3 traffic - if st.getenv('skip_tgen', 'false') != 'true': - st.banner('Step 6: Verifying traffic after rolling reboot') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Traffic verification failed after rolling reboot\n' + # Brief wait for BGP convergence before next reboot + st.wait(10, 'Wait for convergence after {} reboot'.format(node)) + + # Step 6: Verify VTEPs with retries + st.banner('Step 6: Verifying base setup and VTEPs after rolling reboot') + if not verify_base_setup_bgw(setup_nodes, retry=10): + summ += 'Base setup verification failed after rolling reboot\n' result = False - else: - st.log('L2+L3 traffic verified after rolling reboot') - # Step 7: Check for core files - st.banner('Step 7: Checking for core files and crashes') + vtep_nodes = test_cfg['nodes'].get('l2l3vni', []) + if vtep_nodes: + if not vxlan_obj.verify_vtep(vtep_nodes, dci_enabled=True): + summ += 'Remote VTEPs not fully recovered after rolling reboot\n' + result = False + else: + st.log('All remote VTEPs are present') + + finally: + # Stop continuous traffic (not measuring drop on continuous stream) + if fc_available and fc_streams: + st.banner('Stopping continuous traffic (drop not measured on continuous stream)') + try: + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + except Exception: + pass + + # Step 7: Verify burst traffic l2_v4, l2_v6, l3_v4, l3_v6 across and within DC + st.banner('Step 7: Verifying burst traffic across and within DC after rolling reboot') + if not verify_traffic(tgen_handles, bum=True, + traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6']): + summ += 'Traffic verification failed after rolling reboot\n' + result = False + else: + st.log('L2+L3 traffic verified across and within DC after rolling reboot') + + # Step 8: Check for core files + st.banner('Step 8: Checking for core files and crashes') if vxlan_obj.check_core(): summ += 'Core files detected after rolling reboot\n' result = False @@ -7091,24 +7172,25 @@ def test_base_dci_l3vni_rolling_reboot(self): # ------------------------------------------------------------------ # L3VNI_dci:96 – Remove/Add Import Export RT + # Uses continuous traffic; verifies traffic recovers within threshold. # ------------------------------------------------------------------ def test_base_dci_l3vni_remove_add_import_export_rt(self): """ L3VNI_dci:96 - Remove/Add Import Export RT on all BGWs of DC1. Description: - Remove and then re-add the route-target import/export configuration - on all BGW spines of DC1. Verify that cross-DC L3VNI traffic recovers - after the RT is restored. + Start continuous L2+L3 traffic, remove and then re-add the + route-target import/export configuration on all BGW spines + of DC1, then verify the continuous traffic recovers with + drop within threshold. Steps: - 1. Verify base setup before trigger - 2. Verify baseline cross-DC traffic - 3. Remove import/export RT on all DC1 BGWs - 4. Wait for traffic to be impacted - 5. Re-add import/export RT on all DC1 BGWs - 6. Verify traffic recovers - 7. Check for core files and crashes + 1. Verify base setup + 2. Start continuous traffic L2 and L3 (dci_flap_continuous) + 3. Remove/Add Import Export RT on all BGWs of 1 DC + 4. Verify continuous traffic recovers and drop is within threshold + 5. Stop continuous traffic + 6. Check for core files/crashes """ tc_id = 'test_base_dci_l3vni_remove_add_import_export_rt' test_cfg['tc_id'] = tc_id @@ -7117,6 +7199,10 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): st.banner('Testcase L3VNI_dci:96: Remove/Add Import Export RT ({})'.format(tc_id)) result = True summ = '' + stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) + start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) + fc_streams = tgen_handles.get('dci_flap_continuous') + fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) @@ -7137,12 +7223,24 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): report_result(False, tc_id, summ) return - # Step 2: Verify baseline traffic - st.banner('Step 2: Verify baseline cross-DC L2+L3 traffic') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Baseline traffic verification failed before RT removal\n' + # Step 2: Start continuous L2+L3 traffic + st.banner('Step 2: Start continuous L2+L3 traffic (dci_flap_continuous)') + if not fc_available: + summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' + report_result(False, tc_id, summ) + return + + try: + vxlan_obj.check_traffic( + fc_streams, + regenerate_traffic_items=True, + action='start', + stop_proto_wait=stop_pw, + start_proto_wait=start_pw, + ) + except Exception as err: + st.error('Continuous traffic start failed: {}'.format(err)) + summ += 'Continuous traffic start failed: {}\n'.format(err) report_result(False, tc_id, summ) return @@ -7151,7 +7249,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): bgp_info = vxlan_obj.generate_bgp_underlay_info(dci_enabled=True) try: - # Step 3: Remove import/export RT on all DC1 BGWs + # Step 3: Remove/Add Import Export RT on all DC1 BGWs st.banner('Step 3: Remove import/export RT on DC1 BGWs') for bgw in dc1_bgws: st.log('Removing import/export RT on {}'.format(bgw)) @@ -7161,12 +7259,9 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): else: st.log('No RT config generated for {}, skipping'.format(bgw)) - # Step 4: Wait for traffic impact - st.banner('Step 4: Wait for traffic to be impacted by RT removal') st.wait(10, 'Wait for RT removal to take effect') - # Step 5: Re-add import/export RT on all DC1 BGWs - st.banner('Step 5: Re-add import/export RT on DC1 BGWs') + st.banner('Step 3b: Re-add import/export RT on DC1 BGWs') for bgw in dc1_bgws: st.log('Re-adding import/export RT on {}'.format(bgw)) add_cfg = vxlan_obj.add_bgw_import_export_rt(bgw, config_dict, bgp_info) @@ -7175,9 +7270,19 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): else: st.log('No RT config generated for {}, skipping'.format(bgw)) + # Step 4: Verify continuous traffic recovers within threshold + st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') + st.wait(15, 'Wait for BGP convergence after RT re-add') + if not vxlan_obj.check_traffic( + fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): + summ += 'Continuous traffic did not recover after import/export RT remove/add\n' + result = False + else: + st.log('Continuous traffic recovered after import/export RT remove/add: PASS') + except Exception as e: st.error('Exception during RT manipulation: {}'.format(e)) - # Attempt recovery + # Attempt RT recovery for bgw in dc1_bgws: try: add_cfg = vxlan_obj.add_bgw_import_export_rt(bgw, config_dict, bgp_info) @@ -7186,22 +7291,18 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): except Exception: pass summ += 'Exception during RT manipulation: {}\n'.format(e) - report_result(False, tc_id, summ) - return - - # Step 6: Verify traffic recovers - st.banner('Step 6: Wait for convergence and verify traffic recovery') - st.wait(15, 'Wait for BGP convergence after RT re-add') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Traffic did not recover after import/export RT remove/add\n' result = False - else: - st.log('Traffic recovered after import/export RT remove/add: PASS') - # Step 7: Check for core files - st.banner('Step 7: Checking for core files and crashes') + finally: + # Step 5: Stop continuous traffic + st.banner('Step 5: Stopping continuous traffic') + try: + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + except Exception: + pass + + # Step 6: Check for core files + st.banner('Step 6: Checking for core files and crashes') if vxlan_obj.check_core(): summ += 'Core files detected after import/export RT remove/add\n' result = False @@ -7210,25 +7311,26 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): # ------------------------------------------------------------------ # L3VNI_dci:97 – Remove/Add RT_REWRITE configs + # Uses continuous traffic; verifies traffic recovers within threshold. # ------------------------------------------------------------------ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): """ L3VNI_dci:97 - Remove/Add RT_REWRITE configs on all BGWs of DC1. Description: - Remove and then re-add the RT-REWRITE route-maps (RT-REWRITE-WAN, - RT-REWRITE-DC) and associated extcommunity-lists on all BGW spines - of DC1. Verify that cross-DC L3VNI traffic recovers after configs - are restored. + Start continuous L2+L3 traffic, remove and then re-add the + RT-REWRITE route-maps (RT-REWRITE-WAN, RT-REWRITE-DC) and + associated extcommunity-lists on all BGW spines of DC1, + then verify the continuous traffic recovers with drop + within threshold. Steps: - 1. Verify base setup before trigger - 2. Verify baseline cross-DC traffic - 3. Remove RT-REWRITE configs on all DC1 BGWs - 4. Wait for traffic to be impacted - 5. Re-add RT-REWRITE configs on all DC1 BGWs - 6. Verify traffic recovers - 7. Check for core files and crashes + 1. Verify base setup + 2. Start continuous traffic L2 and L3 (dci_flap_continuous) + 3. Remove/Add RT-REWRITE configs on all BGWs of 1 DC + 4. Verify continuous traffic recovers and drop is within threshold + 5. Stop continuous traffic + 6. Check for core files/crashes """ tc_id = 'test_base_dci_l3vni_remove_add_rt_rewrite_configs' test_cfg['tc_id'] = tc_id @@ -7237,6 +7339,10 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): st.banner('Testcase L3VNI_dci:97: Remove/Add RT_REWRITE configs ({})'.format(tc_id)) result = True summ = '' + stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) + start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) + fc_streams = tgen_handles.get('dci_flap_continuous') + fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) @@ -7257,12 +7363,24 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): report_result(False, tc_id, summ) return - # Step 2: Verify baseline traffic - st.banner('Step 2: Verify baseline cross-DC L2+L3 traffic') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Baseline traffic verification failed before RT-REWRITE removal\n' + # Step 2: Start continuous L2+L3 traffic + st.banner('Step 2: Start continuous L2+L3 traffic (dci_flap_continuous)') + if not fc_available: + summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' + report_result(False, tc_id, summ) + return + + try: + vxlan_obj.check_traffic( + fc_streams, + regenerate_traffic_items=True, + action='start', + stop_proto_wait=stop_pw, + start_proto_wait=start_pw, + ) + except Exception as err: + st.error('Continuous traffic start failed: {}'.format(err)) + summ += 'Continuous traffic start failed: {}\n'.format(err) report_result(False, tc_id, summ) return @@ -7272,7 +7390,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): dci_vip_maps = vxlan_obj.generate_dci_vip_maps() try: - # Step 3: Remove RT-REWRITE configs on all DC1 BGWs + # Step 3: Remove/Add RT-REWRITE configs on all DC1 BGWs st.banner('Step 3: Remove RT-REWRITE configs on DC1 BGWs') for bgw in dc1_bgws: st.log('Removing RT-REWRITE configs on {}'.format(bgw)) @@ -7282,12 +7400,9 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): else: st.log('No RT-REWRITE config generated for {}, skipping'.format(bgw)) - # Step 4: Wait for traffic impact - st.banner('Step 4: Wait for traffic to be impacted by RT-REWRITE removal') st.wait(10, 'Wait for RT-REWRITE removal to take effect') - # Step 5: Re-add RT-REWRITE configs on all DC1 BGWs - st.banner('Step 5: Re-add RT-REWRITE configs on DC1 BGWs') + st.banner('Step 3b: Re-add RT-REWRITE configs on DC1 BGWs') for bgw in dc1_bgws: st.log('Re-adding RT-REWRITE configs on {}'.format(bgw)) add_cfg = vxlan_obj.add_bgw_rt_rewrite_maps(bgw, config_dict, bgp_info, dci_vip_maps) @@ -7296,6 +7411,16 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): else: st.log('No RT-REWRITE config generated for {}, skipping'.format(bgw)) + # Step 4: Verify continuous traffic recovers within threshold + st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') + st.wait(15, 'Wait for BGP convergence after RT-REWRITE re-add') + if not vxlan_obj.check_traffic( + fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): + summ += 'Continuous traffic did not recover after RT-REWRITE remove/add\n' + result = False + else: + st.log('Continuous traffic recovered after RT-REWRITE remove/add: PASS') + except Exception as e: st.error('Exception during RT-REWRITE manipulation: {}'.format(e)) # Attempt recovery @@ -7307,22 +7432,18 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): except Exception: pass summ += 'Exception during RT-REWRITE manipulation: {}\n'.format(e) - report_result(False, tc_id, summ) - return - - # Step 6: Verify traffic recovers - st.banner('Step 6: Wait for convergence and verify traffic recovery') - st.wait(15, 'Wait for BGP convergence after RT-REWRITE re-add') - if not verify_traffic(tgen_handles, bum=True, - traffic_types=['l2_v4', 'l2_v6', 'l3_v4', 'l3_v6'], - scope='cross'): - summ += 'Traffic did not recover after RT-REWRITE remove/add\n' result = False - else: - st.log('Traffic recovered after RT-REWRITE remove/add: PASS') - # Step 7: Check for core files - st.banner('Step 7: Checking for core files and crashes') + finally: + # Step 5: Stop continuous traffic + st.banner('Step 5: Stopping continuous traffic') + try: + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + except Exception: + pass + + # Step 6: Check for core files + st.banner('Step 6: Checking for core files and crashes') if vxlan_obj.check_core(): summ += 'Core files detected after RT-REWRITE remove/add\n' result = False From 5cc4e9dd90600370cd5b2097e7c0a9a4ecd67666 Mon Sep 17 00:00:00 2001 From: bhavani Date: Thu, 9 Apr 2026 23:59:11 +0000 Subject: [PATCH 3/6] Fix 5 reviewer issues: PortChannel discovery, stream filtering, rate config, stream disable, DF/NDF verification 1. Fix PortChannel discovery for DC2 leafs - use test_cfg['port_channels'] pattern instead of config_dict which returned empty lists 2. Filter dci_flap_continuous streams by IP version (v4 or v6) for TC57/58 so only matching streams run per test case 3. Disable continuous streams after stopping in all 6 TCs so they are not left enabled between test executions 4. Configure rate_percent=0.01 on continuous streams before starting in all 6 TCs 5. Use 'show evpn es' (get_evpn_es) to determine DF/NDF status before shutdown and print output after PortChannel shutdown in TC57/58 6. Store 'name' and 'version' fields in stream_handles dict (vxlan_helper.py) to enable stream filtering by IP version --- spytest/tests/csco-tmp/test_vxlan_dci.py | 188 +++++++++++++++++++---- spytest/tests/csco-tmp/vxlan_helper.py | 2 + 2 files changed, 162 insertions(+), 28 deletions(-) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index f0d5c8a40be..94517797560 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -6701,7 +6701,7 @@ def test_base_dci_multicast_traffic(self): # ------------------------------------------------------------------ # L3VNI_dci:57 / L3VNI_dci:58 – DF Failover with L3VNI traffic # Uses continuous cross-DC traffic (dci_flap_continuous) per - # InterfaceTrigger pattern. + # InterfaceTrigger pattern. Only v4 streams for TC57, v6 for TC58. # ------------------------------------------------------------------ @pytest.mark.parametrize("ip_version", ["v4", "v6"]) def test_base_dci_l3vni_mh_df_failover(self, ip_version): @@ -6711,21 +6711,23 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): Description: Multi-homed host on DC2 (leaf0_dc2 + leaf1_dc2 form an EVPN MH pair). One leaf is the Designated Forwarder (DF) and the other is Non-DF. - Start continuous cross-DC traffic, shutdown the DF leaf's host-facing - PortChannel to trigger DF election to the peer leaf, then verify - continuous traffic continues without significant loss. + Start continuous cross-DC traffic (only the matching IP version), + shutdown the DF leaf's host-facing PortChannel to trigger DF + election to the peer leaf, then verify continuous traffic continues + without significant loss. Steps: 1. Verify base setup - 2. Pick Leaf0 and Leaf1 of DC2 (one DF, one NDF) - 3. Start continuous cross-DC traffic (dci_flap_continuous) + 2. Pick Leaf0 and Leaf1 of DC2; use 'show evpn es' to identify DF/NDF + 3. Start continuous cross-DC traffic (filtered to v4 or v6 only) 4. Shutdown DF link of multi-homed host - 5. Verify DF election moves to peer leaf, continuous traffic continues - 6. Restore PortChannel (no-shut), stop continuous traffic + 5. Print 'show evpn es' after shutdown; verify continuous traffic + 6. Restore PortChannel (no-shut), stop and disable continuous traffic 7. Check for core files and crashes """ tc_num = 57 if ip_version == 'v4' else 58 ip_label = 'IPv4' if ip_version == 'v4' else 'IPv6' + ixia_version = 'ipv4' if ip_version == 'v4' else 'ipv6' tc_id = 'test_base_dci_l3vni_mh_{}_df_failover'.format(ip_version) test_cfg['tc_id'] = tc_id tc_cfg = vxlan_obj.get_tc_params(tc_id) @@ -6736,8 +6738,18 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): summ = '' stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) - fc_streams = tgen_handles.get('dci_flap_continuous') + + # Get all dci_flap_continuous streams and filter to the requested IP version only + all_fc_streams = tgen_handles.get('dci_flap_continuous') + fc_streams = {} + if isinstance(all_fc_streams, dict): + for k, v in all_fc_streams.items(): + if isinstance(v, dict) and v.get('version', '') == ixia_version: + fc_streams[k] = v fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 + st.log('dci_flap_continuous {} streams: {} (total: {})'.format( + ip_version, len(fc_streams), + len(all_fc_streams) if isinstance(all_fc_streams, dict) else 0)) # Step 1: Verify base setup st.banner('Step 1: Verify base setup before DF failover') @@ -6747,7 +6759,7 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): report_result(False, tc_id, summ) return - # Step 2: Pick Leaf0 and Leaf1 of DC2 (DF/NDF pair) + # Step 2: Pick Leaf0 and Leaf1 of DC2 and determine DF/NDF via 'show evpn es' st.banner('Step 2: Pick Leaf0 and Leaf1 of DC2 as DF/NDF pair') dc2_leafs = [n for n in test_cfg['nodes'].get('l2l3vni_bgw', []) if 'leaf' in n and 'dc2' in n] @@ -6761,40 +6773,70 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): dc2_leafs = sorted(dc2_leafs)[:2] st.log('DC2 MH leaf pair: {}'.format(dc2_leafs)) - # Get PortChannel interfaces on each DC2 leaf - config_dict = vxlan_obj.get_cfg_dict() + # Get PortChannel interfaces from test_cfg (port_channels key) pc_map = {} for leaf in dc2_leafs: - leaf_cfg = config_dict.get(leaf, {}) pcs = [] - if isinstance(leaf_cfg, dict) and leaf_cfg.get('port_channel'): - for pc_item in leaf_cfg['port_channel']: - pc_name = pc_item.get('name', '') - if pc_name: - pcs.append(pc_name) - if not pcs: - int_cfg = vxlan_obj.get_config_interfaces_list(st.get_testbed_vars()) - l2vni_intfs = int_cfg.get(leaf, {}).get('l2vni_int', []) - pcs = [i for i in l2vni_intfs if i.startswith('PortChannel')] + for pc in (test_cfg.get(leaf, {}).get('port_channels', []) or []): + pc_num = pc.get('port_channel_num') + if pc_num is not None: + pcs.append('PortChannel{}'.format(pc_num)) pc_map[leaf] = pcs st.log('{} PortChannels: {}'.format(leaf, pcs)) - df_leaf = dc2_leafs[0] - ndf_leaf = dc2_leafs[1] + # Use 'show evpn es' to determine DF vs NDF + # 'N' in Type column means non-DF for that ESI + df_leaf = None + ndf_leaf = None + for leaf in dc2_leafs: + es_output = vxlan_obj.get_evpn_es(leaf) + st.log('{} show evpn es output: {}'.format(leaf, es_output)) + for es_entry in (es_output or []): + es_type = es_entry.get('type', '') + es_if = es_entry.get('es_if', '') + if es_if.startswith('PortChannel') and 'N' in es_type: + ndf_leaf = leaf + elif es_if.startswith('PortChannel') and 'L' in es_type and 'N' not in es_type: + df_leaf = leaf + + # Fallback: if we could not determine DF/NDF, pick first leaf with PCs as DF + if not df_leaf: + for leaf in dc2_leafs: + if pc_map.get(leaf): + df_leaf = leaf + break + if not ndf_leaf: + ndf_leaf = [l for l in dc2_leafs if l != df_leaf][0] if df_leaf else dc2_leafs[1] + if not df_leaf: + df_leaf = dc2_leafs[0] + df_pcs = pc_map.get(df_leaf, []) if not df_pcs: summ += 'No PortChannels found on DF leaf {}\n'.format(df_leaf) report_result(False, tc_id, summ) return - st.log('DF leaf: {} (PortChannels: {}), Peer leaf: {}'.format(df_leaf, df_pcs, ndf_leaf)) + st.log('DF leaf: {} (PortChannels: {}), NDF leaf: {}'.format(df_leaf, df_pcs, ndf_leaf)) - # Step 3: Start continuous cross-DC traffic + # Step 3: Start continuous cross-DC traffic (filtered to ip_version) st.banner('Step 3: Start continuous {} cross-DC traffic (dci_flap_continuous)'.format(ip_label)) if not fc_available: - summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' + summ += 'dci_flap_continuous {} streams not available, cannot run continuous traffic test\n'.format(ip_version) report_result(False, tc_id, summ) return + # Configure rate_percent to 0.01 on selected streams before starting + tg_handle = None + fc_stream_ids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + fc_stream_ids.append(_v['stream_id']) + if not tg_handle: + tg_handle = _v.get('tg_handle') + if tg_handle and fc_stream_ids: + for sid in fc_stream_ids: + tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) + st.log('Configured rate_percent=0.01 on {} continuous streams'.format(len(fc_stream_ids))) + try: vxlan_obj.check_traffic( fc_streams, @@ -6817,6 +6859,12 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): intf_obj.interface_shutdown(dut=df_leaf, interfaces=pc) st.wait(10, 'Wait for DF election to move to peer leaf') + # Print 'show evpn es' after shutdown to confirm DF/NDF change + st.banner('Step 4b: Print show evpn es after PortChannel shutdown') + for leaf in dc2_leafs: + es_output = vxlan_obj.get_evpn_es(leaf) + st.log('{} show evpn es (after shutdown): {}'.format(leaf, es_output)) + # Step 5: Verify continuous traffic continues after DF failover st.banner('Step 5: Verify continuous {} cross-DC traffic after DF failover'.format(ip_label)) if not vxlan_obj.check_traffic( @@ -6827,7 +6875,7 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): st.log('Continuous {} cross-DC traffic continues after DF failover: PASS'.format(ip_label)) finally: - # Step 6: Restore PortChannel and stop continuous traffic + # Step 6: Restore PortChannel, stop and disable continuous traffic st.banner('Step 6: Restore PortChannel on DF leaf {} and stop continuous traffic'.format(df_leaf)) for pc in df_pcs: st.log('No-shutting PortChannel {} on {}'.format(pc, df_leaf)) @@ -6836,6 +6884,12 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass + # Disable continuous streams so they are not left enabled + if tg_handle and fc_stream_ids: + try: + tg_handle.tg_traffic_config(mode='disable', stream_id=fc_stream_ids) + except Exception: + pass st.wait(15, 'Wait for PortChannel and DF election recovery') # Step 7: Check for core files and crashes @@ -6916,7 +6970,20 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): # Step 3: Start continuous L2+L3 traffic across DC st.banner('Step 3: Start continuous L2+L3 traffic across DC (dci_flap_continuous)') + # Collect stream IDs and tg_handle for rate config and later disable + _fc_tg_handle = None + _fc_stream_ids = [] if fc_available: + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _fc_stream_ids.append(_v['stream_id']) + if not _fc_tg_handle: + _fc_tg_handle = _v.get('tg_handle') + # Configure rate_percent to 0.01 on continuous streams + if _fc_tg_handle and _fc_stream_ids: + for sid in _fc_stream_ids: + _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) + st.log('Configured rate_percent=0.01 on {} continuous streams'.format(len(_fc_stream_ids))) try: vxlan_obj.check_traffic( fc_streams, @@ -6987,6 +7054,12 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass + # Disable continuous streams so they are not left enabled + if _fc_tg_handle and _fc_stream_ids: + try: + _fc_tg_handle.tg_traffic_config(mode='disable', stream_id=_fc_stream_ids) + except Exception: + pass # Step 7: Verify burst traffic l2_v4, l2_v6, l3_v4, l3_v6 across and within DC st.banner('Step 7: Verifying burst traffic across and within DC after simultaneous BGW reboot') @@ -7085,7 +7158,20 @@ def test_base_dci_l3vni_rolling_reboot(self): # Step 3: Start continuous L2+L3 traffic across and within DC st.banner('Step 3: Start continuous L2+L3 traffic across and within DC (dci_flap_continuous)') + # Collect stream IDs and tg_handle for rate config and later disable + _fc_tg_handle = None + _fc_stream_ids = [] if fc_available: + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _fc_stream_ids.append(_v['stream_id']) + if not _fc_tg_handle: + _fc_tg_handle = _v.get('tg_handle') + # Configure rate_percent to 0.01 on continuous streams + if _fc_tg_handle and _fc_stream_ids: + for sid in _fc_stream_ids: + _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) + st.log('Configured rate_percent=0.01 on {} continuous streams'.format(len(_fc_stream_ids))) try: vxlan_obj.check_traffic( fc_streams, @@ -7152,6 +7238,12 @@ def test_base_dci_l3vni_rolling_reboot(self): vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass + # Disable continuous streams so they are not left enabled + if _fc_tg_handle and _fc_stream_ids: + try: + _fc_tg_handle.tg_traffic_config(mode='disable', stream_id=_fc_stream_ids) + except Exception: + pass # Step 7: Verify burst traffic l2_v4, l2_v6, l3_v4, l3_v6 across and within DC st.banner('Step 7: Verifying burst traffic across and within DC after rolling reboot') @@ -7230,6 +7322,20 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): report_result(False, tc_id, summ) return + # Collect stream IDs and tg_handle for rate config and later disable + _fc_tg_handle = None + _fc_stream_ids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _fc_stream_ids.append(_v['stream_id']) + if not _fc_tg_handle: + _fc_tg_handle = _v.get('tg_handle') + # Configure rate_percent to 0.01 on continuous streams + if _fc_tg_handle and _fc_stream_ids: + for sid in _fc_stream_ids: + _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) + st.log('Configured rate_percent=0.01 on {} continuous streams'.format(len(_fc_stream_ids))) + try: vxlan_obj.check_traffic( fc_streams, @@ -7300,6 +7406,12 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass + # Disable continuous streams so they are not left enabled + if _fc_tg_handle and _fc_stream_ids: + try: + _fc_tg_handle.tg_traffic_config(mode='disable', stream_id=_fc_stream_ids) + except Exception: + pass # Step 6: Check for core files st.banner('Step 6: Checking for core files and crashes') @@ -7370,6 +7482,20 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): report_result(False, tc_id, summ) return + # Collect stream IDs and tg_handle for rate config and later disable + _fc_tg_handle = None + _fc_stream_ids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _fc_stream_ids.append(_v['stream_id']) + if not _fc_tg_handle: + _fc_tg_handle = _v.get('tg_handle') + # Configure rate_percent to 0.01 on continuous streams + if _fc_tg_handle and _fc_stream_ids: + for sid in _fc_stream_ids: + _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) + st.log('Configured rate_percent=0.01 on {} continuous streams'.format(len(_fc_stream_ids))) + try: vxlan_obj.check_traffic( fc_streams, @@ -7441,6 +7567,12 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass + # Disable continuous streams so they are not left enabled + if _fc_tg_handle and _fc_stream_ids: + try: + _fc_tg_handle.tg_traffic_config(mode='disable', stream_id=_fc_stream_ids) + except Exception: + pass # Step 6: Check for core files st.banner('Step 6: Checking for core files and crashes') diff --git a/spytest/tests/csco-tmp/vxlan_helper.py b/spytest/tests/csco-tmp/vxlan_helper.py index a4cf84747a5..e4e3ab6433b 100644 --- a/spytest/tests/csco-tmp/vxlan_helper.py +++ b/spytest/tests/csco-tmp/vxlan_helper.py @@ -3208,6 +3208,8 @@ def create_traffic_item(device_handles, endpoints, topo_handles, transmit_mode=" stream_handles[i]['tg_handle'] = tg_handle stream_handles[i]['port_handle'] = port_handle stream_handles[i]['dst_ports'] = emulation_dst_port + stream_handles[i]['name'] = name + stream_handles[i]['version'] = version i+=1 st.wait(1) return stream_handles From f1a448f8f02c1f52712e02e8b08a964015114d26 Mon Sep 17 00:00:00 2001 From: bhavani Date: Fri, 10 Apr 2026 00:40:36 +0000 Subject: [PATCH 4/6] Disable all continuous streams at TC start to prevent streams from being enabled before test execution All 6 TCs now explicitly stop+disable all dci_flap_continuous streams at the beginning of each test case, before any traffic operations. This ensures streams are only enabled when check_traffic(action='start') is called during the test execution phase. --- spytest/tests/csco-tmp/test_vxlan_dci.py | 85 ++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index 94517797560..accb7a64d39 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -6751,6 +6751,23 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): ip_version, len(fc_streams), len(all_fc_streams) if isinstance(all_fc_streams, dict) else 0)) + # Ensure ALL continuous streams are disabled at the start of this TC + if isinstance(all_fc_streams, dict) and len(all_fc_streams) > 0: + _all_tg = None + _all_sids = [] + for _k, _v in all_fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _all_sids.append(_v['stream_id']) + if not _all_tg: + _all_tg = _v.get('tg_handle') + if _all_tg and _all_sids: + try: + _all_tg.tg_traffic_control(action='stop', stream_handle=_all_sids, max_wait_timer='10') + except Exception: + pass + _all_tg.tg_traffic_config(mode='disable', stream_id=_all_sids) + st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_all_sids))) + # Step 1: Verify base setup st.banner('Step 1: Verify base setup before DF failover') setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) @@ -6939,6 +6956,23 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 + # Ensure all continuous streams are disabled at the start of this TC + if fc_available: + _init_tg = None + _init_sids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _init_sids.append(_v['stream_id']) + if not _init_tg: + _init_tg = _v.get('tg_handle') + if _init_tg and _init_sids: + try: + _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') + except Exception: + pass + _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) + st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) + # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: @@ -7117,6 +7151,23 @@ def test_base_dci_l3vni_rolling_reboot(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 + # Ensure all continuous streams are disabled at the start of this TC + if fc_available: + _init_tg = None + _init_sids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _init_sids.append(_v['stream_id']) + if not _init_tg: + _init_tg = _v.get('tg_handle') + if _init_tg and _init_sids: + try: + _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') + except Exception: + pass + _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) + st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) + # Build ordered list of DC1 nodes for rolling reboot # Order: leaf1_dc1 -> leaf2_dc1 -> spine0_dc1 -> spine1_bgw (first DC1 BGW) all_nodes = test_cfg['nodes'].get('l2l3vni_bgw', []) + test_cfg['nodes'].get('spine', []) @@ -7296,6 +7347,23 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 + # Ensure all continuous streams are disabled at the start of this TC + if fc_available: + _init_tg = None + _init_sids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _init_sids.append(_v['stream_id']) + if not _init_tg: + _init_tg = _v.get('tg_handle') + if _init_tg and _init_sids: + try: + _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') + except Exception: + pass + _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) + st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) + # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: @@ -7456,6 +7524,23 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 + # Ensure all continuous streams are disabled at the start of this TC + if fc_available: + _init_tg = None + _init_sids = [] + for _k, _v in fc_streams.items(): + if isinstance(_v, dict) and _v.get('stream_id'): + _init_sids.append(_v['stream_id']) + if not _init_tg: + _init_tg = _v.get('tg_handle') + if _init_tg and _init_sids: + try: + _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') + except Exception: + pass + _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) + st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) + # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: From 6526379c6f115c334b265ef6fc2205d5b9e33b27 Mon Sep 17 00:00:00 2001 From: bhavani Date: Fri, 10 Apr 2026 02:59:29 +0000 Subject: [PATCH 5/6] Fix PortChannel discovery fallback and TC96/97 stream filtering (1 VLAN per VRF) --- spytest/tests/csco-tmp/test_vxlan_dci.py | 101 +++++++++++++++++++---- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index accb7a64d39..192de127bc1 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -6783,6 +6783,10 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): if len(dc2_leafs) < 2: dc2_leafs = [n for n in test_cfg['nodes'].get('l2l3vni', []) if 'leaf' in n and 'dc2' in n] + if len(dc2_leafs) < 2: + # Fallback: search ALL leaf nodes for DC2 leafs with port_channels + dc2_leafs = [n for n in test_cfg['nodes'].get('leaf', []) + if 'dc2' in n and test_cfg.get(n, {}).get('port_channels')] if len(dc2_leafs) < 2: pytest.skip('DC2 does not have 2 MH leafs for DF failover test') return @@ -6800,6 +6804,9 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): pcs.append('PortChannel{}'.format(pc_num)) pc_map[leaf] = pcs st.log('{} PortChannels: {}'.format(leaf, pcs)) + if not pcs: + st.log('DEBUG: test_cfg keys for {}: {}'.format( + leaf, list(test_cfg.get(leaf, {}).keys()) if test_cfg.get(leaf) else 'NOT IN test_cfg')) # Use 'show evpn es' to determine DF vs NDF # 'N' in Type column means non-DF for that ESI @@ -7383,22 +7390,55 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): report_result(False, tc_id, summ) return - # Step 2: Start continuous L2+L3 traffic - st.banner('Step 2: Start continuous L2+L3 traffic (dci_flap_continuous)') + # Step 2: Start continuous traffic — 1 VLAN per VRF (2 streams only) + st.banner('Step 2: Start continuous traffic (1 VLAN per VRF from dci_flap_continuous)') if not fc_available: summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' report_result(False, tc_id, summ) return + # Filter fc_streams to 1 stream per VRF (2 streams total) + _seen_vrfs = set() + _fc_subset = {} + _fc_subset_key = 1 + for _k, _v in fc_streams.items(): + if not isinstance(_v, dict) or not _v.get('stream_id'): + continue + _sname = _v.get('name', '') + # Extract VRF from stream name (e.g. '_vrf101' or '_vrf102') + _vrf_tag = '' + if '_vrf' in _sname: + for _part in _sname.split('_'): + if _part.startswith('vrf'): + _vrf_tag = _part + break + if not _vrf_tag: + # For streams without VRF in name, derive from VLAN + # VLAN 12 -> VRF 101, VLAN 18 -> VRF 102 + if 'vlan12' in _sname or 'v12' in _sname: + _vrf_tag = 'vrf101' + elif 'vlan18' in _sname or 'v18' in _sname: + _vrf_tag = 'vrf102' + else: + _vrf_tag = 'unknown_{}'.format(_k) + if _vrf_tag not in _seen_vrfs: + _seen_vrfs.add(_vrf_tag) + _fc_subset[_fc_subset_key] = _v + _fc_subset_key += 1 + st.log('TC96 selected stream {}: name={} vrf={}'.format(_k, _sname, _vrf_tag)) + + st.log('TC96: filtered to {} streams (1 per VRF) from {} total'.format( + len(_fc_subset), len(fc_streams))) + # Collect stream IDs and tg_handle for rate config and later disable _fc_tg_handle = None _fc_stream_ids = [] - for _k, _v in fc_streams.items(): + for _k, _v in _fc_subset.items(): if isinstance(_v, dict) and _v.get('stream_id'): _fc_stream_ids.append(_v['stream_id']) if not _fc_tg_handle: _fc_tg_handle = _v.get('tg_handle') - # Configure rate_percent to 0.01 on continuous streams + # Configure rate_percent to 0.01 on selected streams if _fc_tg_handle and _fc_stream_ids: for sid in _fc_stream_ids: _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) @@ -7406,7 +7446,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): try: vxlan_obj.check_traffic( - fc_streams, + _fc_subset, regenerate_traffic_items=True, action='start', stop_proto_wait=stop_pw, @@ -7448,7 +7488,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') st.wait(15, 'Wait for BGP convergence after RT re-add') if not vxlan_obj.check_traffic( - fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): + _fc_subset, action='check', stop_start_protocols=False, min_perc=99.6): summ += 'Continuous traffic did not recover after import/export RT remove/add\n' result = False else: @@ -7471,7 +7511,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): # Step 5: Stop continuous traffic st.banner('Step 5: Stopping continuous traffic') try: - vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + vxlan_obj.check_traffic(_fc_subset, action='stop', stop_start_protocols=False) except Exception: pass # Disable continuous streams so they are not left enabled @@ -7560,22 +7600,55 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): report_result(False, tc_id, summ) return - # Step 2: Start continuous L2+L3 traffic - st.banner('Step 2: Start continuous L2+L3 traffic (dci_flap_continuous)') + # Step 2: Start continuous traffic — 1 VLAN per VRF (2 streams only) + st.banner('Step 2: Start continuous traffic (1 VLAN per VRF from dci_flap_continuous)') if not fc_available: summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' report_result(False, tc_id, summ) return + # Filter fc_streams to 1 stream per VRF (2 streams total) + _seen_vrfs = set() + _fc_subset = {} + _fc_subset_key = 1 + for _k, _v in fc_streams.items(): + if not isinstance(_v, dict) or not _v.get('stream_id'): + continue + _sname = _v.get('name', '') + # Extract VRF from stream name (e.g. '_vrf101' or '_vrf102') + _vrf_tag = '' + if '_vrf' in _sname: + for _part in _sname.split('_'): + if _part.startswith('vrf'): + _vrf_tag = _part + break + if not _vrf_tag: + # For streams without VRF in name, derive from VLAN + # VLAN 12 -> VRF 101, VLAN 18 -> VRF 102 + if 'vlan12' in _sname or 'v12' in _sname: + _vrf_tag = 'vrf101' + elif 'vlan18' in _sname or 'v18' in _sname: + _vrf_tag = 'vrf102' + else: + _vrf_tag = 'unknown_{}'.format(_k) + if _vrf_tag not in _seen_vrfs: + _seen_vrfs.add(_vrf_tag) + _fc_subset[_fc_subset_key] = _v + _fc_subset_key += 1 + st.log('TC97 selected stream {}: name={} vrf={}'.format(_k, _sname, _vrf_tag)) + + st.log('TC97: filtered to {} streams (1 per VRF) from {} total'.format( + len(_fc_subset), len(fc_streams))) + # Collect stream IDs and tg_handle for rate config and later disable _fc_tg_handle = None _fc_stream_ids = [] - for _k, _v in fc_streams.items(): + for _k, _v in _fc_subset.items(): if isinstance(_v, dict) and _v.get('stream_id'): _fc_stream_ids.append(_v['stream_id']) if not _fc_tg_handle: _fc_tg_handle = _v.get('tg_handle') - # Configure rate_percent to 0.01 on continuous streams + # Configure rate_percent to 0.01 on selected streams if _fc_tg_handle and _fc_stream_ids: for sid in _fc_stream_ids: _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) @@ -7583,7 +7656,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): try: vxlan_obj.check_traffic( - fc_streams, + _fc_subset, regenerate_traffic_items=True, action='start', stop_proto_wait=stop_pw, @@ -7626,7 +7699,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') st.wait(15, 'Wait for BGP convergence after RT-REWRITE re-add') if not vxlan_obj.check_traffic( - fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): + _fc_subset, action='check', stop_start_protocols=False, min_perc=99.6): summ += 'Continuous traffic did not recover after RT-REWRITE remove/add\n' result = False else: @@ -7649,7 +7722,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): # Step 5: Stop continuous traffic st.banner('Step 5: Stopping continuous traffic') try: - vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) + vxlan_obj.check_traffic(_fc_subset, action='stop', stop_start_protocols=False) except Exception: pass # Disable continuous streams so they are not left enabled From 067a1d5e4b581f78b3160587cb32101ee8d5e592 Mon Sep 17 00:00:00 2001 From: bhavani Date: Fri, 10 Apr 2026 19:42:22 +0000 Subject: [PATCH 6/6] Fix 3 reviewer issues: remove disable-at-TC-start, MH-only continuous streams, TC96/97 L2+L3 streams 1. Remove disable-at-TC-start blocks from all 6 TCs (57/58, 84, 85, 96, 97) - Streams are disabled by default; no need to stop/disable at TC start - This also saves time at the beginning of each TC 2. TC57/58: Filter to PortChannel (MH) sources only + 1 L2 stream + L3 (1 per VRF) - tgen_preconfig: L2 continuous streams now only use l2_pc_cross (dropped orphan) - tgen_preconfig: L3 continuous streams filter to MH-only, 1 VLAN per VRF - TC filtering: Pick 1 L2 + L3 streams (1 per VRF) by parsing stream names 3. TC96/97: Use ALL dci_flap_continuous streams (both L2 AND L3 v4+v6) - Removed VRF-based filtering that limited to 1 stream per VRF (L2 only) - Now starts/checks/stops all L2 + L3 continuous streams --- spytest/tests/csco-tmp/test_vxlan_dci.py | 301 ++++++++--------------- 1 file changed, 103 insertions(+), 198 deletions(-) diff --git a/spytest/tests/csco-tmp/test_vxlan_dci.py b/spytest/tests/csco-tmp/test_vxlan_dci.py index 192de127bc1..6309f6ed4a4 100644 --- a/spytest/tests/csco-tmp/test_vxlan_dci.py +++ b/spytest/tests/csco-tmp/test_vxlan_dci.py @@ -1433,23 +1433,21 @@ def _dci_merge_flap_continuous(dest, created, start_key): return k # Limit flap-continuous streams: two VLANs only (12, 18) — one traffic item per VLAN per AF + # Only PortChannel (MH) sources for continuous streams (DF failover / link flap tests) _dci_fc_vlans = (12, 18) def _dci_fc_endpoints_for_vlan(vlan_id): merged = {} - if l2_orphan_cross: - for _k, _v in l2_orphan_cross.items(): - if _v.get('src_vlan') == vlan_id: - merged[_k] = _v + # Only use PortChannel (MH) cross-DC endpoints for continuous streams if l2_pc_cross: for _k, _v in l2_pc_cross.items(): if _v.get('src_vlan') == vlan_id: merged[_k] = _v return merged - if dci_enabled and (l2_orphan_cross or l2_pc_cross): + if dci_enabled and l2_pc_cross: st.banner( - "DCI: continuous L2 cross-DC for link flap tests — VLANs {} only".format( + "DCI: continuous L2 cross-DC (MH only) for link flap tests — VLANs {} only".format( _dci_fc_vlans)) for _vlan_fc in _dci_fc_vlans: _eps_fc = _dci_fc_endpoints_for_vlan(_vlan_fc) @@ -1460,7 +1458,7 @@ def _dci_fc_endpoints_for_vlan(vlan_id): endpoints=_eps_fc, topo_handles=topo_handles, multi_dst='vlan', - name_prfx='DCI-FC-L2-X-v{}'.format(_vlan_fc), + name_prfx='DCI-FC-L2-MH-X-v{}'.format(_vlan_fc), transmit_mode='continuous', rate_percent=_dci_fc_rate, pkts_per_burst=_dci_fc_ppb, @@ -1476,7 +1474,7 @@ def _dci_fc_endpoints_for_vlan(vlan_id): topo_handles=topo_handles, version='ipv6', multi_dst='vlan', - name_prfx='DCI-FC-L2-X-v{}'.format(_vlan_fc), + name_prfx='DCI-FC-L2-MH-X-v{}'.format(_vlan_fc), transmit_mode='continuous', rate_percent=_dci_fc_rate, pkts_per_burst=_dci_fc_ppb, @@ -1484,31 +1482,47 @@ def _dci_fc_endpoints_for_vlan(vlan_id): _dci_fc_key = _dci_merge_flap_continuous(stream_handles['dci_flap_continuous'], _h_fc, _dci_fc_key) # L3 continuous cross-DC traffic for DCI link flap/shut tests + # Only PortChannel (MH) sources, limited to 1 VLAN per VRF if dci_enabled and l3_cross_dc_endpoints: - st.banner("DCI: continuous L3 cross-DC for link flap tests") - _h_fc_l3 = vxlan_obj.create_traffic_item( - device_handles=v4_device_handles, - endpoints=l3_cross_dc_endpoints, - topo_handles=topo_handles, - multi_dst='vrf', - name_prfx='DCI-FC-L3-X', - transmit_mode='continuous', - rate_percent=_dci_fc_rate, - pkts_per_burst=_dci_fc_ppb, - ) - _dci_fc_key = _dci_merge_flap_continuous(stream_handles['dci_flap_continuous'], _h_fc_l3, _dci_fc_key) - _h_fc_l3_v6 = vxlan_obj.create_traffic_item( - device_handles=v6_device_handles, - endpoints=l3_cross_dc_endpoints, - topo_handles=topo_handles, - version='ipv6', - multi_dst='vrf', - name_prfx='DCI-FC-L3-X', - transmit_mode='continuous', - rate_percent=_dci_fc_rate, - pkts_per_burst=_dci_fc_ppb, - ) - _dci_fc_key = _dci_merge_flap_continuous(stream_handles['dci_flap_continuous'], _h_fc_l3_v6, _dci_fc_key) + _l3_fc_pc_endpoints = {} + _l3_fc_seen_vrfs = set() + for _k, _v in l3_cross_dc_endpoints.items(): + _src_int = _v.get('src_int', '') + _src_vrf = _v.get('src_vrf', '') + if not _src_int.startswith('PortChannel'): + continue + if _src_vrf in _l3_fc_seen_vrfs: + continue + _l3_fc_seen_vrfs.add(_src_vrf) + _l3_fc_pc_endpoints[_k] = _v + if _l3_fc_pc_endpoints: + st.banner("DCI: continuous L3 cross-DC (MH only, 1 VLAN/VRF) — {} endpoints".format( + len(_l3_fc_pc_endpoints))) + _h_fc_l3 = vxlan_obj.create_traffic_item( + device_handles=v4_device_handles, + endpoints=_l3_fc_pc_endpoints, + topo_handles=topo_handles, + multi_dst='vrf', + name_prfx='DCI-FC-L3-MH-X', + transmit_mode='continuous', + rate_percent=_dci_fc_rate, + pkts_per_burst=_dci_fc_ppb, + ) + _dci_fc_key = _dci_merge_flap_continuous(stream_handles['dci_flap_continuous'], _h_fc_l3, _dci_fc_key) + _h_fc_l3_v6 = vxlan_obj.create_traffic_item( + device_handles=v6_device_handles, + endpoints=_l3_fc_pc_endpoints, + topo_handles=topo_handles, + version='ipv6', + multi_dst='vrf', + name_prfx='DCI-FC-L3-MH-X', + transmit_mode='continuous', + rate_percent=_dci_fc_rate, + pkts_per_burst=_dci_fc_ppb, + ) + _dci_fc_key = _dci_merge_flap_continuous(stream_handles['dci_flap_continuous'], _h_fc_l3_v6, _dci_fc_key) + else: + st.log("NOTE: No PortChannel-sourced L3 cross-DC endpoints for continuous streams.") if stream_handles['dci_flap_continuous']: st.log("DCI flap continuous: {} stream(s) created (L2+L3)".format(len(stream_handles['dci_flap_continuous']))) @@ -6739,35 +6753,48 @@ def test_base_dci_l3vni_mh_df_failover(self, ip_version): stop_pw = test_cfg['global'].get('traffic_stop_protocol_sleep', 15) start_pw = test_cfg['global'].get('traffic_start_protocol_sleep', 15) - # Get all dci_flap_continuous streams and filter to the requested IP version only + # Get all dci_flap_continuous streams, filter to IP version, then pick + # 1 L2 stream + L3 streams (1 per VRF) — MH-only sources all_fc_streams = tgen_handles.get('dci_flap_continuous') - fc_streams = {} + _ver_streams = {} if isinstance(all_fc_streams, dict): for k, v in all_fc_streams.items(): if isinstance(v, dict) and v.get('version', '') == ixia_version: - fc_streams[k] = v + _ver_streams[k] = v + # Further filter: 1 L2 stream + L3 (1 per VRF) + fc_streams = {} + _fc_filt_key = 1 + _l2_picked = False + _l3_vrfs_picked = set() + for k, v in _ver_streams.items(): + if not isinstance(v, dict) or not v.get('stream_id'): + continue + _sname = v.get('name', '') + if 'L2' in _sname: + if not _l2_picked: + fc_streams[_fc_filt_key] = v + _fc_filt_key += 1 + _l2_picked = True + st.log('TC{} picked L2 stream {}: name={}'.format(tc_num, k, _sname)) + elif 'L3' in _sname: + _vrf_tag = '' + if '_vrf' in _sname: + for _part in _sname.split('_'): + if _part.startswith('vrf'): + _vrf_tag = _part + break + if not _vrf_tag: + _vrf_tag = 'l3_{}'.format(k) + if _vrf_tag not in _l3_vrfs_picked: + _l3_vrfs_picked.add(_vrf_tag) + fc_streams[_fc_filt_key] = v + _fc_filt_key += 1 + st.log('TC{} picked L3 stream {}: name={} vrf={}'.format(tc_num, k, _sname, _vrf_tag)) fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - st.log('dci_flap_continuous {} streams: {} (total: {})'.format( - ip_version, len(fc_streams), + st.log('TC{}: {} version-filtered -> {} selected (1 L2 + L3/VRF) from {} total'.format( + tc_num, len(_ver_streams), len(fc_streams), len(all_fc_streams) if isinstance(all_fc_streams, dict) else 0)) - # Ensure ALL continuous streams are disabled at the start of this TC - if isinstance(all_fc_streams, dict) and len(all_fc_streams) > 0: - _all_tg = None - _all_sids = [] - for _k, _v in all_fc_streams.items(): - if isinstance(_v, dict) and _v.get('stream_id'): - _all_sids.append(_v['stream_id']) - if not _all_tg: - _all_tg = _v.get('tg_handle') - if _all_tg and _all_sids: - try: - _all_tg.tg_traffic_control(action='stop', stream_handle=_all_sids, max_wait_timer='10') - except Exception: - pass - _all_tg.tg_traffic_config(mode='disable', stream_id=_all_sids) - st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_all_sids))) - # Step 1: Verify base setup st.banner('Step 1: Verify base setup before DF failover') setup_nodes = test_cfg['nodes'].get('l2l3vni_bgw', test_cfg['nodes'].get('l2l3vni', [])) @@ -6963,23 +6990,6 @@ def test_base_dci_l3vni_simultaneous_bgw_reboot(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - # Ensure all continuous streams are disabled at the start of this TC - if fc_available: - _init_tg = None - _init_sids = [] - for _k, _v in fc_streams.items(): - if isinstance(_v, dict) and _v.get('stream_id'): - _init_sids.append(_v['stream_id']) - if not _init_tg: - _init_tg = _v.get('tg_handle') - if _init_tg and _init_sids: - try: - _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') - except Exception: - pass - _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) - st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) - # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: @@ -7158,23 +7168,6 @@ def test_base_dci_l3vni_rolling_reboot(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - # Ensure all continuous streams are disabled at the start of this TC - if fc_available: - _init_tg = None - _init_sids = [] - for _k, _v in fc_streams.items(): - if isinstance(_v, dict) and _v.get('stream_id'): - _init_sids.append(_v['stream_id']) - if not _init_tg: - _init_tg = _v.get('tg_handle') - if _init_tg and _init_sids: - try: - _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') - except Exception: - pass - _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) - st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) - # Build ordered list of DC1 nodes for rolling reboot # Order: leaf1_dc1 -> leaf2_dc1 -> spine0_dc1 -> spine1_bgw (first DC1 BGW) all_nodes = test_cfg['nodes'].get('l2l3vni_bgw', []) + test_cfg['nodes'].get('spine', []) @@ -7354,23 +7347,6 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - # Ensure all continuous streams are disabled at the start of this TC - if fc_available: - _init_tg = None - _init_sids = [] - for _k, _v in fc_streams.items(): - if isinstance(_v, dict) and _v.get('stream_id'): - _init_sids.append(_v['stream_id']) - if not _init_tg: - _init_tg = _v.get('tg_handle') - if _init_tg and _init_sids: - try: - _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') - except Exception: - pass - _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) - st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) - # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: @@ -7390,55 +7366,28 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): report_result(False, tc_id, summ) return - # Step 2: Start continuous traffic — 1 VLAN per VRF (2 streams only) - st.banner('Step 2: Start continuous traffic (1 VLAN per VRF from dci_flap_continuous)') + # Step 2: Start continuous traffic — all L2 + L3 (v4+v6) streams + st.banner('Step 2: Start continuous L2 + L3 traffic (all dci_flap_continuous streams)') if not fc_available: summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' report_result(False, tc_id, summ) return - # Filter fc_streams to 1 stream per VRF (2 streams total) - _seen_vrfs = set() - _fc_subset = {} - _fc_subset_key = 1 + # Use ALL fc_streams (L2 and L3, v4+v6) — no VRF filtering + st.log('TC96: using all {} dci_flap_continuous streams (L2 + L3 v4/v6)'.format(len(fc_streams))) for _k, _v in fc_streams.items(): - if not isinstance(_v, dict) or not _v.get('stream_id'): - continue - _sname = _v.get('name', '') - # Extract VRF from stream name (e.g. '_vrf101' or '_vrf102') - _vrf_tag = '' - if '_vrf' in _sname: - for _part in _sname.split('_'): - if _part.startswith('vrf'): - _vrf_tag = _part - break - if not _vrf_tag: - # For streams without VRF in name, derive from VLAN - # VLAN 12 -> VRF 101, VLAN 18 -> VRF 102 - if 'vlan12' in _sname or 'v12' in _sname: - _vrf_tag = 'vrf101' - elif 'vlan18' in _sname or 'v18' in _sname: - _vrf_tag = 'vrf102' - else: - _vrf_tag = 'unknown_{}'.format(_k) - if _vrf_tag not in _seen_vrfs: - _seen_vrfs.add(_vrf_tag) - _fc_subset[_fc_subset_key] = _v - _fc_subset_key += 1 - st.log('TC96 selected stream {}: name={} vrf={}'.format(_k, _sname, _vrf_tag)) - - st.log('TC96: filtered to {} streams (1 per VRF) from {} total'.format( - len(_fc_subset), len(fc_streams))) + if isinstance(_v, dict): + st.log('TC96 stream {}: name={}'.format(_k, _v.get('name', ''))) # Collect stream IDs and tg_handle for rate config and later disable _fc_tg_handle = None _fc_stream_ids = [] - for _k, _v in _fc_subset.items(): + for _k, _v in fc_streams.items(): if isinstance(_v, dict) and _v.get('stream_id'): _fc_stream_ids.append(_v['stream_id']) if not _fc_tg_handle: _fc_tg_handle = _v.get('tg_handle') - # Configure rate_percent to 0.01 on selected streams + # Configure rate_percent to 0.01 on all continuous streams if _fc_tg_handle and _fc_stream_ids: for sid in _fc_stream_ids: _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) @@ -7446,7 +7395,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): try: vxlan_obj.check_traffic( - _fc_subset, + fc_streams, regenerate_traffic_items=True, action='start', stop_proto_wait=stop_pw, @@ -7488,7 +7437,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') st.wait(15, 'Wait for BGP convergence after RT re-add') if not vxlan_obj.check_traffic( - _fc_subset, action='check', stop_start_protocols=False, min_perc=99.6): + fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): summ += 'Continuous traffic did not recover after import/export RT remove/add\n' result = False else: @@ -7511,7 +7460,7 @@ def test_base_dci_l3vni_remove_add_import_export_rt(self): # Step 5: Stop continuous traffic st.banner('Step 5: Stopping continuous traffic') try: - vxlan_obj.check_traffic(_fc_subset, action='stop', stop_start_protocols=False) + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass # Disable continuous streams so they are not left enabled @@ -7564,23 +7513,6 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): fc_streams = tgen_handles.get('dci_flap_continuous') fc_available = isinstance(fc_streams, dict) and len(fc_streams) > 0 - # Ensure all continuous streams are disabled at the start of this TC - if fc_available: - _init_tg = None - _init_sids = [] - for _k, _v in fc_streams.items(): - if isinstance(_v, dict) and _v.get('stream_id'): - _init_sids.append(_v['stream_id']) - if not _init_tg: - _init_tg = _v.get('tg_handle') - if _init_tg and _init_sids: - try: - _init_tg.tg_traffic_control(action='stop', stream_handle=_init_sids, max_wait_timer='10') - except Exception: - pass - _init_tg.tg_traffic_config(mode='disable', stream_id=_init_sids) - st.log('Disabled all {} dci_flap_continuous streams at TC start'.format(len(_init_sids))) - # Get DC1 BGW nodes dc1_bgws = test_cfg['nodes'].get('dc1_bgw', []) if not dc1_bgws: @@ -7600,55 +7532,28 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): report_result(False, tc_id, summ) return - # Step 2: Start continuous traffic — 1 VLAN per VRF (2 streams only) - st.banner('Step 2: Start continuous traffic (1 VLAN per VRF from dci_flap_continuous)') + # Step 2: Start continuous traffic — all L2 + L3 (v4+v6) streams + st.banner('Step 2: Start continuous L2 + L3 traffic (all dci_flap_continuous streams)') if not fc_available: summ += 'dci_flap_continuous streams not available, cannot run continuous traffic test\n' report_result(False, tc_id, summ) return - # Filter fc_streams to 1 stream per VRF (2 streams total) - _seen_vrfs = set() - _fc_subset = {} - _fc_subset_key = 1 + # Use ALL fc_streams (L2 and L3, v4+v6) — no VRF filtering + st.log('TC97: using all {} dci_flap_continuous streams (L2 + L3 v4/v6)'.format(len(fc_streams))) for _k, _v in fc_streams.items(): - if not isinstance(_v, dict) or not _v.get('stream_id'): - continue - _sname = _v.get('name', '') - # Extract VRF from stream name (e.g. '_vrf101' or '_vrf102') - _vrf_tag = '' - if '_vrf' in _sname: - for _part in _sname.split('_'): - if _part.startswith('vrf'): - _vrf_tag = _part - break - if not _vrf_tag: - # For streams without VRF in name, derive from VLAN - # VLAN 12 -> VRF 101, VLAN 18 -> VRF 102 - if 'vlan12' in _sname or 'v12' in _sname: - _vrf_tag = 'vrf101' - elif 'vlan18' in _sname or 'v18' in _sname: - _vrf_tag = 'vrf102' - else: - _vrf_tag = 'unknown_{}'.format(_k) - if _vrf_tag not in _seen_vrfs: - _seen_vrfs.add(_vrf_tag) - _fc_subset[_fc_subset_key] = _v - _fc_subset_key += 1 - st.log('TC97 selected stream {}: name={} vrf={}'.format(_k, _sname, _vrf_tag)) - - st.log('TC97: filtered to {} streams (1 per VRF) from {} total'.format( - len(_fc_subset), len(fc_streams))) + if isinstance(_v, dict): + st.log('TC97 stream {}: name={}'.format(_k, _v.get('name', ''))) # Collect stream IDs and tg_handle for rate config and later disable _fc_tg_handle = None _fc_stream_ids = [] - for _k, _v in _fc_subset.items(): + for _k, _v in fc_streams.items(): if isinstance(_v, dict) and _v.get('stream_id'): _fc_stream_ids.append(_v['stream_id']) if not _fc_tg_handle: _fc_tg_handle = _v.get('tg_handle') - # Configure rate_percent to 0.01 on selected streams + # Configure rate_percent to 0.01 on all continuous streams if _fc_tg_handle and _fc_stream_ids: for sid in _fc_stream_ids: _fc_tg_handle.tg_traffic_config(mode='modify', stream_id=sid, rate_percent=0.01) @@ -7656,7 +7561,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): try: vxlan_obj.check_traffic( - _fc_subset, + fc_streams, regenerate_traffic_items=True, action='start', stop_proto_wait=stop_pw, @@ -7699,7 +7604,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): st.banner('Step 4: Verify continuous traffic recovers and drop is within threshold') st.wait(15, 'Wait for BGP convergence after RT-REWRITE re-add') if not vxlan_obj.check_traffic( - _fc_subset, action='check', stop_start_protocols=False, min_perc=99.6): + fc_streams, action='check', stop_start_protocols=False, min_perc=99.6): summ += 'Continuous traffic did not recover after RT-REWRITE remove/add\n' result = False else: @@ -7722,7 +7627,7 @@ def test_base_dci_l3vni_remove_add_rt_rewrite_configs(self): # Step 5: Stop continuous traffic st.banner('Step 5: Stopping continuous traffic') try: - vxlan_obj.check_traffic(_fc_subset, action='stop', stop_start_protocols=False) + vxlan_obj.check_traffic(fc_streams, action='stop', stop_start_protocols=False) except Exception: pass # Disable continuous streams so they are not left enabled