From a000450f2e953caf3d0f8b0039c17ecba07c3f7d Mon Sep 17 00:00:00 2001 From: Mark Xiao Date: Fri, 30 Jan 2026 04:49:33 -0800 Subject: [PATCH] Fix telemetry/test_events.py for v6 topos (#21649) Summary: Fix the following telemetry events for v6 topos: sonic-events-bgp:notification: Utilize the fix for mgmt ipv6 only setup that uses ip6tables on v6 neighbors for v6 topos. Delete the condition for mgmt ipv6 only setup because v4 neighbors still exist in that setup. sonic-events-bgp:bgp-state: Consider v6 neighbors in is_bgp_state_idle() dhcp-relay events: dhcpmon doesn't support v6 topo because it requires ipv4 addr configured on VLAN interface, skip the test for v6 topo Pass tbinfo to trigger functions, currently only drop_tcp_packets uses it, but more functions (e.g., dhcp-relay if dhcpmon is fixed) will need it in the future Fixes #21648 Signed-off-by: markxiao Signed-off-by: mssonicbld --- tests/common/devices/sonic.py | 6 +++-- tests/telemetry/conftest.py | 4 +-- tests/telemetry/events/bgp_events.py | 26 +++++++++--------- tests/telemetry/events/dhcp-relay_events.py | 23 +++++++++------- tests/telemetry/events/eventd_events.py | 4 +-- tests/telemetry/events/host_events.py | 30 ++++++++++----------- tests/telemetry/events/run_events_test.py | 6 ++--- tests/telemetry/events/swss_events.py | 14 +++++----- tests/telemetry/test_events.py | 7 ++--- 9 files changed, 63 insertions(+), 57 deletions(-) diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index d626598ed10..7764cf80b82 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -2193,7 +2193,9 @@ def is_bgp_state_idle(self): Returns: True or False """ - bgp_summary = self.command("show ip bgp summary")["stdout_lines"] + bgp_summary_v4 = self.command("show ip bgp summary")["stdout_lines"] + bgp_summary_v6 = self.command("show ipv6 bgp summary")["stdout_lines"] + bgp_summary = bgp_summary_v4 + bgp_summary_v6 idle_count = 0 expected_idle_count = 0 @@ -2204,7 +2206,7 @@ def is_bgp_state_idle(self): if "Total number of neighbors" in line: tokens = line.split() - expected_idle_count = int(tokens[-1]) + expected_idle_count += int(tokens[-1]) if "BGPMonitor" in line: bgp_monitor_count += 1 diff --git a/tests/telemetry/conftest.py b/tests/telemetry/conftest.py index baa7d4894dc..33f07efbfa9 100644 --- a/tests/telemetry/conftest.py +++ b/tests/telemetry/conftest.py @@ -74,7 +74,7 @@ def do_init(duthost): @pytest.fixture(scope="module") -def test_eventd_healthy(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, +def test_eventd_healthy(duthosts, tbinfo, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, setup_streaming_telemetry, gnxi_path): """ @summary: Test eventd heartbeat before testing all testcases @@ -97,6 +97,6 @@ def test_eventd_healthy(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptf py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "eventd"), "eventd not started.") - module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, None) + module.test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, DATA_DIR, None) logger.info("Completed test file: {}".format("eventd_events test completed.")) diff --git a/tests/telemetry/events/bgp_events.py b/tests/telemetry/events/bgp_events.py index 2155d09b679..b2836b0986c 100644 --- a/tests/telemetry/events/bgp_events.py +++ b/tests/telemetry/events/bgp_events.py @@ -5,38 +5,38 @@ import ipaddress from run_events_test import run_test +from tests.common.utilities import is_ipv6_only_topology logger = logging.getLogger(__name__) tag = "sonic-events-bgp" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, drop_tcp_packets, +def test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, drop_tcp_packets, "bgp_notification.json", "sonic-events-bgp:notification", tag) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_bgp_neighbors, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_bgp_neighbors, "bgp_state.json", "sonic-events-bgp:bgp-state", tag) -def drop_tcp_packets(duthost): - # Check if DUT management is IPv6-only and select appropriate BGP neighbor - dut_facts = duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'] - is_mgmt_ipv6_only = dut_facts.get('is_mgmt_ipv6_only', False) +def drop_tcp_packets(duthost, tbinfo): + # Check if topo is IPv6-only and select appropriate BGP neighbor + is_v6_topo = is_ipv6_only_topology(tbinfo) - # Get all BGP neighbors and filter by IP version based on management interface + # Get all BGP neighbors and filter by IP version based on v6/non-v6 topo all_bgp_neighbors = duthost.get_bgp_neighbors() bgp_neighbor = None - if is_mgmt_ipv6_only: + if is_v6_topo: # Find an IPv6 BGP neighbor for neighbor_ip in all_bgp_neighbors.keys(): if ipaddress.ip_address(neighbor_ip).version == 6: bgp_neighbor = neighbor_ip break if bgp_neighbor is None: - raise Exception("No IPv6 BGP neighbors found for IPv6-only management interface") + raise Exception("No IPv6 BGP neighbors found for IPv6-only topo") iptables_cmd = "ip6tables" logger.info( - "Using IPv6 BGP neighbor %s and ip6tables for IPv6-only DUT management interface", + "Using IPv6 BGP neighbor %s and ip6tables for IPv6-only topo", bgp_neighbor ) else: @@ -49,7 +49,7 @@ def drop_tcp_packets(duthost): # Fallback to first neighbor if no IPv4 found bgp_neighbor = list(all_bgp_neighbors.keys())[0] iptables_cmd = "iptables" - logger.info("Using IPv4 BGP neighbor {} and iptables for IPv4 DUT management interface".format(bgp_neighbor)) + logger.info("Using IPv4 BGP neighbor {} and iptables".format(bgp_neighbor)) holdtime_timer_ms = duthost.get_bgp_neighbor_info(bgp_neighbor)["bgpTimerConfiguredHoldTimeMsecs"] @@ -73,7 +73,7 @@ def drop_tcp_packets(duthost): assert ret["rc"] == 0, "Unable to remove DROP rule from {}".format(iptables_cmd) -def shutdown_bgp_neighbors(duthost): +def shutdown_bgp_neighbors(duthost, tbinfo): logger.info("Shutting down bgp neighbors to test bgp-state event") assert duthost.is_service_running("bgpcfgd", "bgp") is True and duthost.is_bgp_state_idle() is False logger.info("Start all bgp sessions") diff --git a/tests/telemetry/events/dhcp-relay_events.py b/tests/telemetry/events/dhcp-relay_events.py index f5072260f9e..3bab339df6c 100644 --- a/tests/telemetry/events/dhcp-relay_events.py +++ b/tests/telemetry/events/dhcp-relay_events.py @@ -7,6 +7,7 @@ from tests.common.helpers.assertions import pytest_assert as py_assert from tests.common.utilities import wait_until +from tests.common.utilities import is_ipv6_only_topology from run_events_test import run_test from event_utils import find_test_vlan, find_test_client_port_and_mac, create_dhcp_discover_packet @@ -14,7 +15,7 @@ tag = "sonic-events-dhcp-relay" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): if duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'].get("is_smartswitch") and \ duthost.facts.get("router_type") == 'leafrouter': pytest.skip("Skipping dhcp_relay events for smartswitch t1 topologies") @@ -25,30 +26,32 @@ def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang) switch_role = device_metadata['localhost'].get('type', '') if switch_role == 'BmcMgmtToRRouter': pytest.skip("Skipping dhcp_relay events for mx topologies") + if is_ipv6_only_topology(tbinfo): + pytest.skip("Skipping dhcp_relay events for IPv6-only topologies") logger.info("Beginning to test dhcp-relay events") - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_discard, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_discard, "dhcp_relay_discard.json", "sonic-events-dhcp-relay:dhcp-relay-discard", tag, False, 30, ptfadapter) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_disparity, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_disparity, "dhcp_relay_disparity.json", "sonic-events-dhcp-relay:dhcp-relay-disparity", tag, False, 30, ptfadapter) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_bind_failure, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_bind_failure, "dhcp_relay_bind_failure.json", "sonic-events-dhcp-relay:dhcp-relay-bind-failure", tag, False, 30) -def trigger_dhcp_relay_discard(duthost, ptfadapter): - send_dhcp_discover_packets(duthost, ptfadapter) +def trigger_dhcp_relay_discard(duthost, tbinfo, ptfadapter): + send_dhcp_discover_packets(duthost, tbinfo, ptfadapter) -def trigger_dhcp_relay_disparity(duthost, ptfadapter): +def trigger_dhcp_relay_disparity(duthost, tbinfo, ptfadapter): """11 packets because dhcpmon process will store up to 10 unhealthy status events https://github.com/sonic-net/sonic-dhcpmon/blob/master/src/dhcp_mon.cpp#L94 static int dhcp_unhealthy_max_count = 10; Sending at interval of 18 seconds because dhcpmon process will check health at that interval static int window_interval_sec = 18; """ - send_dhcp_discover_packets(duthost, ptfadapter, 11, 18) + send_dhcp_discover_packets(duthost, tbinfo, ptfadapter, 11, 18) -def trigger_dhcp_relay_bind_failure(duthost): +def trigger_dhcp_relay_bind_failure(duthost, tbinfo): # Flush ipv6 vlan address and restart dhc6relay process py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "dhcp_relay"), "dhcp_relay container not started") @@ -84,7 +87,7 @@ def trigger_dhcp_relay_bind_failure(duthost): "dhcp_relay not started.") -def send_dhcp_discover_packets(duthost, ptfadapter, packets_to_send=5, interval=1): +def send_dhcp_discover_packets(duthost, tbinfo, ptfadapter, packets_to_send=5, interval=1): py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "dhcp_relay"), "dhcp_relay container not started") diff --git a/tests/telemetry/events/eventd_events.py b/tests/telemetry/events/eventd_events.py index 366230d14f1..5fdd6591423 100644 --- a/tests/telemetry/events/eventd_events.py +++ b/tests/telemetry/events/eventd_events.py @@ -8,7 +8,7 @@ tag = "sonic-events-eventd" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): logger.info("Beginning to test eventd heartbeat") - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, None, "heartbeat.json", "sonic-events-eventd:heartbeat", tag, True) diff --git a/tests/telemetry/events/host_events.py b/tests/telemetry/events/host_events.py index 88c916da1fa..d13f67e0073 100644 --- a/tests/telemetry/events/host_events.py +++ b/tests/telemetry/events/host_events.py @@ -13,11 +13,11 @@ tag = "sonic-events-host" -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): logger.info("Beginning to test host events") - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_kernel_event, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_kernel_event, "event_kernel.json", "sonic-events-host:event-kernel", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, kill_critical_process, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, kill_critical_process, "process_exited_unexpectedly.json", "sonic-events-host:process-exited-unexpectedly", tag, False) backup_monit_config(duthost) @@ -29,17 +29,17 @@ def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang) ] ) try: - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, None, "memory_usage.json", "sonic-events-host:memory-usage", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, None, "disk_usage.json", "sonic-events-host:disk-usage", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, None, "cpu_usage.json", "sonic-events-host:cpu-usage", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_mem_threshold_exceeded_alert, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_mem_threshold_exceeded_alert, "mem_threshold_exceeded.json", "sonic-events-host:mem-threshold-exceeded", tag) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, restart_container, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, restart_container, "event_stopped_ctr.json", "sonic-events-host:event-stopped-ctr", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, stop_container, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, stop_container, "event_down_ctr.json", "sonic-events-host:event-down-ctr", tag, False) finally: restore_monit_config(duthost) @@ -47,13 +47,13 @@ def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang) try: # We need to alot flat 60 seconds for watchdog timeout to fire since the timer is set to 60\ # With a base limit of 30 seconds, we will use 90 seconds - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, None, "watchdog_timeout.json", "sonic-events-host:watchdog-timeout", tag, False, 90) finally: delete_test_watchdog_timeout_service(duthost) -def trigger_mem_threshold_exceeded_alert(duthost): +def trigger_mem_threshold_exceeded_alert(duthost, tbinfo): logger.info("Invoking memory checker with low threshold") cmd = "docker images | grep -w sonic-gnmi" if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0: @@ -62,7 +62,7 @@ def trigger_mem_threshold_exceeded_alert(duthost): duthost.shell("/usr/bin/memory_checker telemetry 100", module_ignore_errors=True) -def trigger_kernel_event(duthost): +def trigger_kernel_event(duthost, tbinfo): logger.info("Invoking logger for kernel events") # syslog at github.com/torvalds/linux/blob/master/fs/squashfs/decompressor_multi.c#L193 trigger_logger(duthost, "zlib decompression failed, data probably corrupt", "kernel") @@ -95,7 +95,7 @@ def get_critical_process(duthost): return "", "" -def restart_container(duthost): +def restart_container(duthost, tbinfo): logger.info("Stopping container for event stopped event") container = get_running_container(duthost) assert container != "", "No available container for testing" @@ -105,7 +105,7 @@ def restart_container(duthost): assert is_container_running, "{} not running after restart".format(container) -def stop_container(duthost): +def stop_container(duthost, tbinfo): logger.info("Stop container for event down event") container = get_running_container(duthost) assert container != "", "No available container for testing" @@ -125,7 +125,7 @@ def stop_container(duthost): duthost.shell("systemctl restart {}".format(container)) -def kill_critical_process(duthost): +def kill_critical_process(duthost, tbinfo): logger.info("Killing critical process for exited unexpectedly event") pid, container = get_critical_process(duthost) assert pid != "", "No available process for testing" diff --git a/tests/telemetry/events/run_events_test.py b/tests/telemetry/events/run_events_test.py index ba9c6b4fa09..edc8bf175de 100644 --- a/tests/telemetry/events/run_events_test.py +++ b/tests/telemetry/events/run_events_test.py @@ -10,14 +10,14 @@ logger = logging.getLogger(__name__) -def run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger, json_file, +def run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger, json_file, filter_event_regex, tag, heartbeat=False, timeout=30, ptfadapter=None): op_file = os.path.join(data_dir, json_file) if trigger is not None: # no trigger for heartbeat if ptfadapter is None: - trigger(duthost) # add events to cache + trigger(duthost, tbinfo) # add events to cache else: - trigger(duthost, ptfadapter) + trigger(duthost, tbinfo, ptfadapter) listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, timeout) # listen from cache data = {} diff --git a/tests/telemetry/events/swss_events.py b/tests/telemetry/events/swss_events.py index e1ecfd8ee4c..38c3d3ee748 100644 --- a/tests/telemetry/events/swss_events.py +++ b/tests/telemetry/events/swss_events.py @@ -33,9 +33,9 @@ WAIT_TIME = 3 -def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): +def test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): logger.info("Beginning to test swss events") - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_interface, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_interface, "if_state.json", "sonic-events-swss:if-state", tag) asic_type = duthost.facts["asic_type"] @@ -49,14 +49,14 @@ def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang) skip_pfc_hwskus = [] if duthost.facts["hwsku"] not in skip_pfc_hwskus: - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, generate_pfc_storm, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, generate_pfc_storm, "pfc_storm.json", "sonic-events-swss:pfc-storm", tag) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_crm_threshold_exceeded, + run_test(duthost, tbinfo, gnxi_path, ptfhost, data_dir, validate_yang, trigger_crm_threshold_exceeded, "chk_crm_threshold.json", "sonic-events-swss:chk_crm_threshold", tag) -def shutdown_interface(duthost): +def shutdown_interface(duthost, tbinfo): logger.info("Shutting down interface") interfaces = duthost.get_interfaces_status() pattern = re.compile(r'^Ethernet[0-9]{1,2}$') @@ -80,7 +80,7 @@ def shutdown_interface(duthost): wait_until(15, 1, 0, verify_port_admin_oper_status, duthost, if_state_test_port, "up") -def generate_pfc_storm(duthost): +def generate_pfc_storm(duthost, tbinfo): logger.info("Generating pfc storm") interfaces = duthost.get_interfaces_status() pattern = re.compile(r'^Ethernet[0-9]{1,2}$') @@ -102,7 +102,7 @@ def generate_pfc_storm(duthost): format(queue_oid)) -def trigger_crm_threshold_exceeded(duthost): +def trigger_crm_threshold_exceeded(duthost, tbinfo): logger.info("Triggering crm threshold exceeded") duthost.shell("crm config polling interval {}".format(CRM_TEST_POLLING_INTERVAL)) diff --git a/tests/telemetry/test_events.py b/tests/telemetry/test_events.py index 69790244f9b..d2a4fe24e8a 100644 --- a/tests/telemetry/test_events.py +++ b/tests/telemetry/test_events.py @@ -36,8 +36,9 @@ def validate_yang(duthost, op_file="", yang_file=""): @pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) @pytest.mark.disable_loganalyzer -def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, setup_streaming_telemetry, gnxi_path, - test_eventd_healthy, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_host_m, # noqa: F811 +def test_events(duthosts, tbinfo, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, + setup_streaming_telemetry, gnxi_path, test_eventd_healthy, + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_host_m, # noqa: F811 setup_standby_ports_on_non_enum_rand_one_per_hwsku_host_m): # noqa: F811 """ Run series of events inside duthost and validate that output is correct and conforms to YANG schema""" @@ -51,7 +52,7 @@ def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, if file.endswith("_events.py") and not file.endswith("eventd_events.py"): module = __import__(file[:len(file)-3]) try: - module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, validate_yang) + module.test_event(duthost, tbinfo, gnxi_path, ptfhost, ptfadapter, DATA_DIR, validate_yang) except pytest.skip.Exception as e: logger.info("Skipping test file: {} due to {}".format(file, e)) continue