diff --git a/sonic-chassisd/scripts/chassisd b/sonic-chassisd/scripts/chassisd index deb409c38..1722ef523 100755 --- a/sonic-chassisd/scripts/chassisd +++ b/sonic-chassisd/scripts/chassisd @@ -81,7 +81,6 @@ CHASSIS_MODULE_REBOOT_TIMESTAMP_FIELD = 'timestamp' CHASSIS_MODULE_REBOOT_REBOOT_FIELD = 'reboot' DEFAULT_LINECARD_REBOOT_TIMEOUT = 180 DEFAULT_DPU_REBOOT_TIMEOUT = 360 -MAX_DPU_REBOOT_DURATION = 800 PLATFORM_ENV_CONF_FILE = "/usr/share/sonic/platform/platform_env.conf" PLATFORM_JSON_FILE = "/usr/share/sonic/platform/platform.json" @@ -162,6 +161,9 @@ DPU_STATE_MANUAL_INTERVENTION = 'ManualIntervention' DPU_STATE_ADMIN_DOWN = 'AdminDown' DPU_STATE_UNRECOVERABLE = 'Unrecoverable' +# DPU boot_id fields +BOOT_ID = 'boot_id' +BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id' class DpuStates(NamedTuple): """Container for DPU plane states to avoid positional confusion.""" @@ -695,7 +697,7 @@ class ModuleUpdater(logger.Logger): # Chassis app db cleanup of all asics of the module # Get the module key and host name from down_modules key - module, lc = re.split('\|', module_host) + module, lc = re.split(r'\|', module_host) if lc == '': # Host name is not available for this module. No clean up is needed @@ -756,10 +758,10 @@ class SmartSwitchModuleUpdater(ModuleUpdater): self.chassis = chassis self.num_modules = self.chassis.get_num_modules() # Connect to STATE_DB and create chassis info tables - state_db = daemon_base.db_connect("STATE_DB") - self.chassis_table = swsscommon.Table(state_db, CHASSIS_INFO_TABLE) - self.module_table = swsscommon.Table(state_db, CHASSIS_MODULE_INFO_TABLE) - self.midplane_table = swsscommon.Table(state_db, CHASSIS_MIDPLANE_INFO_TABLE) + self.state_db = daemon_base.db_connect("STATE_DB") + self.chassis_table = swsscommon.Table(self.state_db, CHASSIS_INFO_TABLE) + self.module_table = swsscommon.Table(self.state_db, CHASSIS_MODULE_INFO_TABLE) + self.midplane_table = swsscommon.Table(self.state_db, CHASSIS_MIDPLANE_INFO_TABLE) self.info_dict_keys = [CHASSIS_MODULE_INFO_NAME_FIELD, CHASSIS_MODULE_INFO_DESC_FIELD, CHASSIS_MODULE_INFO_SLOT_FIELD, @@ -813,6 +815,9 @@ class SmartSwitchModuleUpdater(ModuleUpdater): 'boot_start_time': time.time(), } + # DPU boot_id tracking + self.dpu_boot_id = {} + def deinit(self): """ Destructor of ModuleUpdater @@ -844,24 +849,66 @@ class SmartSwitchModuleUpdater(ModuleUpdater): else: return ModuleBase.MODULE_STATUS_EMPTY - def retrieve_dpu_reboot_info(self, module): + def retrieve_dpu_reboot_info(self, module_name): """ - Retrieve the most recent reboot cause and time from previous-reboot-cause.json. - Returns (cause_string, time_string), or (None, None) if unavailable. + Retrieve the most recent reboot cause, time and boot_id from + previous-reboot-cause.json. + Returns (cause_string, time_string, boot_id), or (None, None, None) + if unavailable. """ try: - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "previous-reboot-cause.json") + path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module_name.lower(), "previous-reboot-cause.json") if os.path.exists(path): with open(path, 'r') as f: data = json.load(f) cause = data.get("cause") time_str = data.get("name") # Format: "YYYY_MM_DD_HH_MM_SS" - return cause, time_str + boot_id = data.get("boot_id") + return cause, time_str, boot_id else: - self.log_debug(f"{module}: previous-reboot-cause.json not found") + self.log_debug(f"{module_name}: previous-reboot-cause.json not found") + except Exception as e: + self.log_error(f"{module_name}: Failed to read previous-reboot-cause.json: {e}") + return None, None, None + + def _load_persisted_boot_ids(self): + """Load boot_id from each DPU's previous-reboot-cause.json.""" + for module_index in range(0, self.num_modules): + module = self.chassis.get_module(module_index) + module_name = try_get(module.get_name) + _, _, boot_id = self.retrieve_dpu_reboot_info(module_name) + if boot_id: + self.dpu_boot_id[module_name] = boot_id + + def check_dpu_boot_id_change(self, module_name, current_boot_id): + """ + Capture the reboot cause for a single DPU when its boot_id changes. + + Args: + module_name: DPU name, e.g. "DPU0" (the DPU_STATE row key). + current_boot_id: boot_id reported for that DPU in CHASSIS_STATE_DB. + """ + try: + if not current_boot_id: + return + + # No change since the last captured boot -> nothing to do. + if self.dpu_boot_id.get(module_name) == current_boot_id: + return + + module_index = try_get(self.chassis.get_module_index, module_name, default=INVALID_MODULE_INDEX) + if module_index < 0: + self.log_error("Unable to get module-index for {} to capture reboot cause".format(module_name)) + return + module = self.chassis.get_module(module_index) + + self.log_notice("{}: new boot_id {} detected, capturing reboot cause".format(module_name, current_boot_id)) + reboot_cause = try_get(module.get_reboot_cause) + self.persist_dpu_reboot_cause(reboot_cause, module_name, boot_id=current_boot_id) + self.update_dpu_reboot_cause_to_db(module_name) + self.dpu_boot_id[module_name] = current_boot_id except Exception as e: - self.log_error(f"{module}: Failed to read previous-reboot-cause.json: {e}") - return None, None + self.log_error("Failed to process boot_id for {}: {}".format(module_name, e)) def module_db_update(self): for module_index in range(0, self.num_modules): @@ -878,55 +925,8 @@ class SmartSwitchModuleUpdater(ModuleUpdater): (CHASSIS_MODULE_INFO_SLOT_FIELD, module_info_dict[CHASSIS_MODULE_INFO_SLOT_FIELD]), (CHASSIS_MODULE_INFO_OPERSTATUS_FIELD, module_info_dict[CHASSIS_MODULE_INFO_OPERSTATUS_FIELD]), (CHASSIS_MODULE_INFO_SERIAL_FIELD, module_info_dict[CHASSIS_MODULE_INFO_SERIAL_FIELD])]) - - # Get a copy of the previous operational status of the module - prev_status = self.get_module_current_status(key) self.module_table.set(key, fvs) - # Get a copy of the current operational status of the module - current_status = module_info_dict[CHASSIS_MODULE_INFO_OPERSTATUS_FIELD] - - # Operational status transitioning to offline - if prev_status != ModuleBase.MODULE_STATUS_EMPTY and prev_status != str(ModuleBase.MODULE_STATUS_OFFLINE) and current_status == str(ModuleBase.MODULE_STATUS_OFFLINE): - self.log_notice("{} operational status transitioning to offline".format(key)) - - # Persist dpu down time - self.persist_dpu_reboot_time(key) - # persist reboot cause - reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause) - self.persist_dpu_reboot_cause(reboot_cause, key) - # publish reboot cause to db - self.update_dpu_reboot_cause_to_db(key) - - elif (prev_status == ModuleBase.MODULE_STATUS_EMPTY or prev_status == str(ModuleBase.MODULE_STATUS_OFFLINE)) and current_status != str(ModuleBase.MODULE_STATUS_OFFLINE): - self.log_notice(f"{key} operational status transitioning to online") - - reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause) - if isinstance(reboot_cause, (tuple, list)): - current_cause = reboot_cause[0] - else: - current_cause = reboot_cause - - stored_cause, stored_time_str = self.retrieve_dpu_reboot_info(key) - - is_reboot = False - if current_cause and stored_cause and stored_time_str: - try: - stored_dt = datetime.strptime(stored_time_str, "%Y_%m_%d_%H_%M_%S").replace(tzinfo=timezone.utc) - now = datetime.now(timezone.utc) - delta_sec = (now - stored_dt).total_seconds() - - if current_cause == stored_cause and delta_sec < MAX_DPU_REBOOT_DURATION: - self.log_info(f"{key}: is_reboot=True — same reboot cause within {int(delta_sec)}s") - is_reboot = True - except Exception as e: - self.log_error(f"{key}: Reboot cause/time comparison failed: {e}") - - if not is_reboot and (stored_time_str is not None or self._is_first_boot(key)): - # persist reboot cause and publish to db - self.persist_dpu_reboot_cause(reboot_cause, key) - self.update_dpu_reboot_cause_to_db(key) - def _get_module_info(self, module_index): """ Retrieves module info of this module @@ -950,10 +950,79 @@ class SmartSwitchModuleUpdater(ModuleUpdater): return module_info_dict - def update_dpu_state(self, key, state): + def _resolve_midplane_down_reason(self, module, module_name): + """ + Build the dpu_midplane_link_reason for an up->down midplane transition. + + Planned : a graceful admin operation set the transition flag, so the + 'transition_type' field is read from STATE_DB + CHASSIS_MODULE_TABLE and reported as Planned: ''. + Unplanned : no transition flag is set, so the platform is queried via + get_midplane_down_reason() and reported as Unplanned: ''. + """ + # If the reason is stored in a file, return it. + cached = self._read_midplane_down_reason(module_name) + if cached: + return cached + + reason_str = None + # Planned operation (graceful shutdown/startup/reset) in progress? + if try_get(module.get_module_state_transition, module_name, default=False): + try: + module_key = "{}|{}".format(CHASSIS_MODULE_INFO_TABLE, module_name.upper()) + transition_type = self.state_db.hget(module_key, "transition_type") + reason_str = "Planned: '{}'".format(transition_type or "unknown") + except Exception as e: + self.log_error("{}: failed to read transition_type: {}".format(module_name, e)) + + if reason_str is None: + # Unplanned down: ask the platform for the hardware reason. + reason = try_get(module.get_midplane_down_reason, default=None) + if isinstance(reason, (tuple, list)): + parts = [str(p) for p in reason if p not in (None, "")] + detail = ", ".join(parts) if parts else "Unknown" + else: + detail = reason or "Unknown" + reason_str = "Unplanned: '{}'".format(detail) + + self._write_midplane_down_reason(module_name, reason_str) + return reason_str + + def _midplane_reason_path(self, module_name): + return os.path.join(MODULE_REBOOT_CAUSE_DIR, module_name.lower(), "midplane-down-reason.txt") + + def _read_midplane_down_reason(self, module_name): + try: + with open(self._midplane_reason_path(module_name)) as f: + return f.read().strip() or None + except FileNotFoundError: + return None + except Exception as e: + self.log_error("{}: read midplane reason failed: {}".format(module_name, e)) + return None + + def _write_midplane_down_reason(self, module_name, reason): + path = self._midplane_reason_path(module_name) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write((reason or "") + "\n") + except Exception as e: + self.log_error("{}: persist midplane reason failed: {}".format(module_name, e)) + + def _clear_midplane_down_reason(self, module_name): + try: + os.remove(self._midplane_reason_path(module_name)) + except FileNotFoundError: + pass + except Exception as e: + self.log_error("{}: clear midplane reason failed: {}".format(module_name, e)) + + def update_dpu_state(self, key, state, reason=None): """ Update specific DPU state fields in chassisStateDB using the given key. - If state is 'down', set control plane, data plane states to down as well. + If state is 'down', set control plane, data plane states to down as well + and record the midplane-down reason. """ try: # Connect to the CHASSIS_STATE_DB using daemon_base @@ -961,10 +1030,11 @@ class SmartSwitchModuleUpdater(ModuleUpdater): self.chassis_state_db = daemon_base.db_connect("CHASSIS_STATE_DB") - # Prepare the fields to update + # Prepare the fields to update. Coerce reason to "" so the 'up' + # path (reason=None) never writes a None into the DB. updates = { "dpu_midplane_link_state": state, - "dpu_midplane_link_reason": "", + "dpu_midplane_link_reason": reason or "", "dpu_midplane_link_time": get_formatted_time(), } # If midplane state is down, set control plane, data plane states to down as well @@ -1011,38 +1081,8 @@ class SmartSwitchModuleUpdater(ModuleUpdater): """Generates the full path for history files.""" return os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "history", file_name) - def _is_first_boot(self, module): - """Checks if the reboot-cause file indicates a first boot.""" - file_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "reboot-cause.txt") - - try: - with open(file_path, 'r') as f: - content = f.read().strip() - return content == "First boot" - except FileNotFoundError: - return False - - def persist_dpu_reboot_time(self, module): - """Persist the current reboot time to a file.""" - time_str = self._get_current_time_str() - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write(time_str) - - def retrieve_dpu_reboot_time(self, module): - """Retrieve the persisted reboot time from a file.""" - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - - try: - with open(path, 'r') as f: - return f.read().strip() - except FileNotFoundError: - return None - - def persist_dpu_reboot_cause(self, reboot_cause, module): - """Persist the reboot cause information and handle file rotation.""" + def persist_dpu_reboot_cause(self, reboot_cause, module, boot_id=None): + """Persist the reboot cause information, boot_id and handle file rotation.""" # Extract cause and comment from the reboot_cause if reboot_cause: try: @@ -1055,19 +1095,11 @@ class SmartSwitchModuleUpdater(ModuleUpdater): else: cause, comment = "Unknown", "N/A" - prev_reboot_time = self.retrieve_dpu_reboot_time(module) - if prev_reboot_time is None: - prev_reboot_time = self._get_current_time_str() - - file_name = f"{prev_reboot_time}_reboot_cause.json" - prev_reboot_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - - if os.path.exists(prev_reboot_path): - os.remove(prev_reboot_path) - + reboot_time = self._get_current_time_str() + file_name = f"{reboot_time}_reboot_cause.json" file_path = self._get_history_path(module, file_name) try: - formatted_time = get_formatted_time(datetimeobj=datetime.strptime(prev_reboot_time, "%Y_%m_%d_%H_%M_%S")) + formatted_time = get_formatted_time(datetimeobj=datetime.strptime(reboot_time, "%Y_%m_%d_%H_%M_%S")) except ValueError: formatted_time = get_formatted_time() @@ -1076,7 +1108,8 @@ class SmartSwitchModuleUpdater(ModuleUpdater): "comment": comment, "device": module, "time": formatted_time, - "name": prev_reboot_time, + "name": reboot_time, + "boot_id": boot_id or "", } with open(file_path, 'w') as f: @@ -1188,10 +1221,13 @@ class SmartSwitchModuleUpdater(ModuleUpdater): # Update midplane state in the chassisStateDB DPU_STATE table key = DPU_STATE_TABLE + "|" + module_key dpu_mp_state = self.get_dpu_midplane_state(key) - if midplane_access and dpu_mp_state != 'up': - self.update_dpu_state(key, 'up') + if midplane_access: + if dpu_mp_state != 'up': + self.update_dpu_state(key, "up") + self._clear_midplane_down_reason(module_key) elif not midplane_access and dpu_mp_state != 'down': - self.update_dpu_state(key, "down") + midplane_down_reason = self._resolve_midplane_down_reason(module, module_key) + self.update_dpu_state(key, "down", midplane_down_reason) # Update db with midplane information fvs = swsscommon.FieldValuePairs([(CHASSIS_MIDPLANE_INFO_IP_FIELD, midplane_ip), @@ -1713,6 +1749,57 @@ class SmartSwitchConfigManagerTask(ProcessTaskBase): self.config_updater.module_config_update(key, admin_state) + +# +# Reboot-cause subscriber task ============================================= +# + + +class RebootCauseSubscriberTask(ProcessTaskBase): + """ Capture DPU reboot-cause on a boot_id change. """ + + def __init__(self, log_identifier, module_updater): + super(RebootCauseSubscriberTask, self).__init__() + self.logger = logger.Logger(log_identifier) + self.module_updater = module_updater + + def task_worker(self): + self.chassis_state_db = daemon_base.db_connect("CHASSIS_STATE_DB") + self.module_updater.chassis_state_db = daemon_base.db_connect("CHASSIS_STATE_DB") + self.module_updater._load_persisted_boot_ids() + + sel = swsscommon.Select() + sst = swsscommon.SubscriberStateTable(self.chassis_state_db, "DPU_STATE") + sel.addSelectable(sst) + + # Listen indefinitely for DPU_STATE changes in CHASSIS_STATE_DB + try: + while True: + # Use timeout so SIGTERM (handled in the parent) can stop us + (state, c) = sel.select(SELECT_TIMEOUT) + + if state == swsscommon.Select.TIMEOUT: + continue + if state != swsscommon.Select.OBJECT: + self.logger.log_warning("sel.select() did not return swsscommon.Select.OBJECT") + continue + + result = sst.pop() + if result is None: + continue + (key, op, fvp) = result + if op != "SET" or fvp is None: + continue + + fvp_dict = dict(fvp) + # check if the boot_id has changed for the DPU + if BOOT_ID in fvp_dict: + self.module_updater.check_dpu_boot_id_change(key, fvp_dict[BOOT_ID]) + + except KeyboardInterrupt: + pass + + # # State Manager task ======================================================== # @@ -1780,14 +1867,24 @@ class DpuStateUpdater(logger.Logger): self.dpu_state_table.hset(self.name, CP_STATE, state) self.dpu_state_table.hset(self.name, CP_UPDATE_TIME, self._time_now()) + def _update_boot_id(self, boot_id): + self.dpu_state_table.hset(self.name, BOOT_ID, boot_id) + def get_dp_state(self): return 'up' if self._get_dp_state() else 'down' def get_cp_state(self): return 'up' if self._get_cp_state() else 'down' - def update_state(self): + def get_boot_id(self): + try: + with open(BOOT_ID_PATH) as f: + return f.read().strip() + except OSError as err: + self.log_warning(f"Failed to read boot_id from {BOOT_ID_PATH}: {err}") + return None + def update_state(self): dp_current_state = self.get_dp_state() _, dp_prev_state = self.dpu_state_table.hget(self.name, DP_STATE) @@ -1799,7 +1896,14 @@ class DpuStateUpdater(logger.Logger): if cp_current_state != cp_prev_state: self._update_cp_dpu_state(cp_current_state) - return [dp_current_state, cp_current_state] + + current_boot_id = self.get_boot_id() + if current_boot_id: + _, prev_boot_id = self.dpu_state_table.hget(self.name, BOOT_ID) + if current_boot_id != prev_boot_id: + self._update_boot_id(current_boot_id) + + return [dp_current_state, cp_current_state, current_boot_id] def deinit(self): self._update_dp_dpu_state('down') @@ -1912,18 +2016,21 @@ class ChassisdDaemon(daemon_base.DaemonBase): self.log_error("Chassisd not supported for this platform") sys.exit(CHASSIS_NOT_SUPPORTED) + self.config_manager = None + self.reboot_cause_subscriber = None + try: - # Start configuration manager task + # Start configuration manager and reboot cause subscriber tasks if self.smartswitch: self.set_initial_dpu_admin_state() self.module_updater.init_dpu_recovery_state() self.config_manager = SmartSwitchConfigManagerTask() self.config_manager.task_run() + self.reboot_cause_subscriber = RebootCauseSubscriberTask(SYSLOG_IDENTIFIER, self.module_updater) + self.reboot_cause_subscriber.task_run() elif self.module_updater.supervisor_slot == self.module_updater.my_slot: self.config_manager = ConfigManagerTask() self.config_manager.task_run() - else: - self.config_manager = None # Start main loop self.log_info("Start daemon main loop") @@ -1943,6 +2050,8 @@ class ChassisdDaemon(daemon_base.DaemonBase): # https://github.com/sonic-net/sonic-buildimage/issues/24775 if self.config_manager is not None: self.config_manager.task_stop() + if self.reboot_cause_subscriber is not None: + self.reboot_cause_subscriber.task_stop() # Delete all the information from DB and then exit self.module_updater.deinit() @@ -1962,7 +2071,8 @@ class DpuStateManagerTask(ProcessTaskBase): self.chassis_state_db = daemon_base.db_connect('CHASSIS_STATE_DB') self.current_dp_state = None self.current_cp_state = None - + self.current_boot_id = self.dpu_state_updater.get_boot_id() + def task_worker(self): sel = swsscommon.Select() selectable = [ @@ -1974,6 +2084,10 @@ class DpuStateManagerTask(ProcessTaskBase): for s in selectable: sel.addSelectable(s) + # write boot_id into DPU_STATE table before entering the loop + if self.current_boot_id: + self.dpu_state_updater._update_boot_id(self.current_boot_id) + try: while True: (state, c) = sel.select(SELECT_TIMEOUT) @@ -2002,7 +2116,8 @@ class DpuStateManagerTask(ProcessTaskBase): fvs = dict(fvp) # No need to update if the state is the same as the current state if ('dpu_data_plane_state' in fvs and fvs['dpu_data_plane_state'] == self.current_dp_state) and \ - ('dpu_control_plane_state' in fvs and fvs['dpu_control_plane_state'] == self.current_cp_state): + ('dpu_control_plane_state' in fvs and fvs['dpu_control_plane_state'] == self.current_cp_state) and \ + ('boot_id' in fvs and fvs['boot_id'] == self.current_boot_id): update_required = False continue self.logger.log_info(f"DPU_STATE change detected: operation={op}, key={key}") @@ -2012,7 +2127,7 @@ class DpuStateManagerTask(ProcessTaskBase): break if update_required: - [self.current_dp_state, self.current_cp_state] = self.dpu_state_updater.update_state() + [self.current_dp_state, self.current_cp_state, self.current_boot_id] = self.dpu_state_updater.update_state() except KeyboardInterrupt: pass diff --git a/sonic-chassisd/tests/mock_platform.py b/sonic-chassisd/tests/mock_platform.py index fb157f21d..90b684e5e 100644 --- a/sonic-chassisd/tests/mock_platform.py +++ b/sonic-chassisd/tests/mock_platform.py @@ -34,6 +34,8 @@ def __init__(self, module_index, module_name, module_desc, module_type, module_s self.admin_state = 1 self.supervisor_slot = 16 self.midplane_access = False + self.state_transition = False + self.state_transition_type = None self.asic_list = asic_list self.module_serial = module_serial @@ -84,15 +86,18 @@ def set_admin_state_gracefully(self, up): def clear_module_state_transition(self, module_name): """Mock implementation of clear_module_state_transition""" + self.state_transition = False return True def set_module_state_transition(self, module_name, transition_type): """Mock implementation of set_module_state_transition""" + self.state_transition = True + self.state_transition_type = transition_type return True def get_module_state_transition(self, module_name): """Mock implementation of get_module_state_transition""" - return False + return self.state_transition def clear_module_gnoi_halt_in_progress(self): """Mock implementation of clear_module_gnoi_halt_in_progress""" @@ -112,6 +117,12 @@ def is_midplane_reachable(self): def set_midplane_reachable(self, up): self.midplane_access = up + def get_midplane_down_reason(self): + return getattr(self, 'midplane_down_reason', None) + + def set_midplane_down_reason(self, reason): + self.midplane_down_reason = reason + def get_all_asics(self): return self.asic_list diff --git a/sonic-chassisd/tests/test_chassisd.py b/sonic-chassisd/tests/test_chassisd.py index 81c1555b1..e61d39a56 100644 --- a/sonic-chassisd/tests/test_chassisd.py +++ b/sonic-chassisd/tests/test_chassisd.py @@ -5,11 +5,25 @@ import json import pytest import time +import importlib.util +import importlib.machinery + from mock import Mock, MagicMock, patch, mock_open from sonic_py_common import daemon_base +from sonic_platform_base.chassis_base import ChassisBase from .mock_platform import MockChassis, MockSmartSwitchChassis, MockModule from .mock_module_base import ModuleBase + +# imp is deprecated in Python 3.12 +def load_source(module_name, file_path): + loader = importlib.machinery.SourceFileLoader(module_name, file_path) + spec = importlib.util.spec_from_file_location(module_name, file_path, loader=loader) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module # required: `from chassisd import *` relies on this + loader.exec_module(module) + return module + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../scripts")) # Assuming OBJECT should be a specific value, define it manually @@ -221,65 +235,96 @@ def test_smartswitch_moduleupdater_status_transitions(): # Create the updater module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) - # Mock dependent methods - with patch.object(module_updater, 'retrieve_dpu_reboot_info', return_value=("Switch rebooted DPU", "2023_01_01_00_00_00")) as mock_reboot_info, \ - patch.object(module_updater, '_is_first_boot', return_value=False) as mock_is_first_boot, \ - patch.object(module_updater, 'persist_dpu_reboot_cause') as mock_persist_reboot_cause, \ - patch.object(module_updater, 'update_dpu_reboot_cause_to_db') as mock_update_reboot_db, \ - patch("os.makedirs") as mock_makedirs, \ - patch("builtins.open", mock_open()) as mock_file, \ - patch.object(module_updater, '_get_history_path', return_value="/tmp/prev_reboot_time.txt") as mock_get_history_path: - - # Transition from ONLINE to OFFLINE - offline_status = ModuleBase.MODULE_STATUS_OFFLINE - module.set_oper_status(offline_status) - module_updater.module_db_update() - assert module.get_oper_status() == offline_status - - # Reset mocks for next transition - mock_file.reset_mock() - mock_makedirs.reset_mock() - mock_persist_reboot_cause.reset_mock() - mock_update_reboot_db.reset_mock() + # Transition from ONLINE to OFFLINE + offline_status = ModuleBase.MODULE_STATUS_OFFLINE + module.set_oper_status(offline_status) + module_updater.module_db_update() + assert module.get_oper_status() == offline_status - # Ensure ONLINE transition is handled correctly - online_status = ModuleBase.MODULE_STATUS_ONLINE - module.set_oper_status(online_status) - module_updater.module_db_update() - assert module.get_oper_status() == online_status + # Ensure ONLINE transition is handled correctly + online_status = ModuleBase.MODULE_STATUS_ONLINE + module.set_oper_status(online_status) + module_updater.module_db_update() + assert module.get_oper_status() == online_status - # Validate mock calls for ONLINE transition - mock_persist_reboot_cause.assert_called_once() - mock_update_reboot_db.assert_called_once() -def test_online_transition_skips_reboot_update(): +def _make_boot_id_updater(): + """Helper: build a SmartSwitchModuleUpdater with one DPU for boot_id consumer tests.""" chassis = MockSmartSwitchChassis() - index = 0 - name = "DPU0" - module = MockModule(index, name, "DPU", ModuleBase.MODULE_TYPE_DPU, 0, "SN123") - module.set_oper_status(ModuleBase.MODULE_STATUS_OFFLINE) + module = MockModule(0, "DPU0", "DPU Module 0", ModuleBase.MODULE_TYPE_DPU, 0, "DPU0-0000") chassis.module_list.append(module) - updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) + return updater - # Mock the module going ONLINE - module.set_oper_status(ModuleBase.MODULE_STATUS_ONLINE) - with patch.object(updater, 'retrieve_dpu_reboot_info', - return_value=("Switch rebooted DPU", datetime.now(timezone.utc).strftime("%Y_%m_%d_%H_%M_%S"))), \ - patch.object(module, 'get_reboot_cause', return_value="Switch rebooted DPU"), \ - patch.object(updater, '_is_first_boot', return_value=False), \ - patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ - patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update, \ - patch("builtins.open", mock_open()), \ - patch("os.makedirs"), \ - patch.object(updater, '_get_history_path', return_value="/tmp/fake.json"): - - updater.module_db_update() - - # Ensure no reboot update due to is_reboot = True +def test_check_dpu_boot_id_change_new_boot(): + """New boot_id -> reboot cause captured and cache updated.""" + updater = _make_boot_id_updater() + updater.dpu_boot_id = {"DPU0": "old-boot-id"} + + with patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.check_dpu_boot_id_change("DPU0", "new-boot-id") + + mock_persist.assert_called_once() + # boot_id must be forwarded to persist so it lands in the json/db. + assert mock_persist.call_args.kwargs.get("boot_id") == "new-boot-id" + mock_update_db.assert_called_once_with("DPU0") + # Cache advanced so the same boot is not re-captured. + assert updater.dpu_boot_id["DPU0"] == "new-boot-id" + + +def test_check_dpu_boot_id_change_same_boot(): + """Unchanged boot_id -> nothing captured (avoids duplicate on every event).""" + updater = _make_boot_id_updater() + updater.dpu_boot_id = {"DPU0": "same-boot-id"} + + with patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.check_dpu_boot_id_change("DPU0", "same-boot-id") + + mock_persist.assert_not_called() + mock_update_db.assert_not_called() + + +@pytest.mark.parametrize("boot_id", [None, ""]) +def test_check_dpu_boot_id_change_no_boot_id(boot_id): + """Empty/None boot_id -> nothing captured.""" + updater = _make_boot_id_updater() + updater.dpu_boot_id = {} + + with patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.check_dpu_boot_id_change("DPU0", boot_id) + mock_persist.assert_not_called() - mock_update.assert_not_called() + mock_update_db.assert_not_called() + + +def test_check_dpu_boot_id_change_unknown_module(): + """Unknown DPU name (no module index) -> nothing captured.""" + updater = _make_boot_id_updater() + updater.dpu_boot_id = {} + + with patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.check_dpu_boot_id_change("DPU_NONEXISTENT", "new-boot-id") + + mock_persist.assert_not_called() + mock_update_db.assert_not_called() + + +def test_load_persisted_boot_ids_populates_cache(): + """_load_persisted_boot_ids reads each DPU's persisted boot_id into the cache.""" + updater = _make_boot_id_updater() + updater.dpu_boot_id = {} + + with patch.object(updater, 'retrieve_dpu_reboot_info', + return_value=("Kernel Panic", "2026_05_19_10_00_00", "persisted-boot-id")): + updater._load_persisted_boot_ids() + + assert updater.dpu_boot_id == {"DPU0": "persisted-boot-id"} + def test_retrieve_dpu_reboot_info_success(): class DummyChassis: @@ -287,12 +332,13 @@ def get_num_modules(self): return 0 def init_midplane_switch(self): return False updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis()) - sample_json = {"cause": "Switch rebooted DPU", "name": "2025_06_25_17_18_52"} + sample_json = {"cause": "Switch rebooted DPU", "name": "2025_06_25_17_18_52", "boot_id": "e4252288-be0d-40ec-8338-d1e5ec206771"} with patch("os.path.exists", return_value=True), \ patch("builtins.open", mock_open(read_data=json.dumps(sample_json))): - cause, time_str = updater.retrieve_dpu_reboot_info("dpu0") + cause, time_str, boot_id = updater.retrieve_dpu_reboot_info("dpu0") assert cause == "Switch rebooted DPU" assert time_str == "2025_06_25_17_18_52" + assert boot_id == "e4252288-be0d-40ec-8338-d1e5ec206771" def test_retrieve_dpu_reboot_info_file_missing(): class DummyChassis: @@ -301,9 +347,62 @@ def init_midplane_switch(self): return False # required for SmartSwitchModuleUp updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis()) with patch("os.path.exists", return_value=False): - cause, time_str = updater.retrieve_dpu_reboot_info("dpu0") + cause, time_str, boot_id = updater.retrieve_dpu_reboot_info("dpu0") assert cause is None assert time_str is None + assert boot_id is None + + +def test_reboot_cause_subscriber_processes_boot_id(): + """Subscriber initializes and forwards a valid boot_id event.""" + module_updater = MagicMock(spec=SmartSwitchModuleUpdater) + subscriber = RebootCauseSubscriberTask(SYSLOG_IDENTIFIER, module_updater) + subscriber_db = MagicMock() + updater_db = MagicMock() + mock_select = MagicMock() + mock_sst = MagicMock() + select_object = swsscommon.Select.OBJECT + select_timeout = swsscommon.Select.TIMEOUT + + mock_select.select.side_effect = [(select_object, None), KeyboardInterrupt] + mock_sst.pop.return_value = ("DPU0", "SET", (("boot_id", "new-boot-id"),)) + + with patch("chassisd.daemon_base.db_connect", + side_effect=[subscriber_db, updater_db]) as mock_db_connect, \ + patch("chassisd.swsscommon.Select", return_value=mock_select) as mock_select_class, \ + patch("chassisd.swsscommon.SubscriberStateTable", return_value=mock_sst) as mock_sst_class: + mock_select_class.TIMEOUT = select_timeout + mock_select_class.OBJECT = select_object + subscriber.task_worker() + + assert mock_db_connect.call_count == 2 + assert module_updater.chassis_state_db is updater_db + module_updater._load_persisted_boot_ids.assert_called_once_with() + mock_sst_class.assert_called_once_with(subscriber_db, "DPU_STATE") + mock_select.addSelectable.assert_called_once_with(mock_sst) + module_updater.check_dpu_boot_id_change.assert_called_once_with("DPU0", "new-boot-id") + + +def test_get_boot_id_reads_kernel_boot_id(): + """get_boot_id returns the stripped kernel boot ID.""" + updater = DpuStateUpdater.__new__(DpuStateUpdater) + updater._syslog = MagicMock() + + with patch("builtins.open", mock_open(read_data="test-boot-id\n")): + assert updater.get_boot_id() == "test-boot-id" + + +def test_get_boot_id_returns_none_on_oserror(): + """get_boot_id returns None and logs a warning when the file cannot be read.""" + updater = DpuStateUpdater.__new__(DpuStateUpdater) + updater._syslog = MagicMock() + updater.log_warning = MagicMock() + + with patch("builtins.open", side_effect=OSError("boot ID unavailable")): + assert updater.get_boot_id() is None + + updater.log_warning.assert_called_once() + def test_smartswitch_moduleupdater_check_invalid_name(): chassis = MockSmartSwitchChassis() @@ -679,40 +778,6 @@ def test_update_dpu_reboot_cause_to_db(mock_open, mock_glob): mock_log_warning.assert_any_call("Error processing file /host/reboot-cause/module/dpu0/history/file1.txt: Unable to read file") -def test_smartswitch_module_db_update(): - chassis = MockSmartSwitchChassis() - reboot_cause = "Power loss" - key = "DPU0" - index = 0 - name = "DPU0" - desc = "DPU Module 0" - slot = 0 - serial = "DPU0-0000" - module_type = ModuleBase.MODULE_TYPE_DPU - module = MockModule(index, name, desc, module_type, slot, serial) - - # Set initial state - status = ModuleBase.MODULE_STATUS_ONLINE - module.set_oper_status(status) - chassis.module_list.append(module) - - module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) - expected_path = "/host/reboot-cause/module/reboot_cause/dpu0/history/2024_11_13_15_06_40_reboot_cause.txt" - symlink_path = "/host/reboot-cause/module/dpu0/previous-reboot-cause.json" - - with patch("os.path.exists", return_value=True), \ - patch("os.makedirs") as mock_makedirs, \ - patch("builtins.open", mock_open(read_data="Power loss")) as mock_file, \ - patch("os.remove") as mock_remove, \ - patch("os.symlink") as mock_symlink: - - # Call the function to test - module_updater.persist_dpu_reboot_cause(reboot_cause, key) - module_updater._is_first_boot(name) - module_updater.persist_dpu_reboot_time(name) - module_updater.update_dpu_reboot_cause_to_db(name) - - def test_platform_json_file_exists_and_valid(): """Test case where the platform JSON file exists with valid data.""" chassis = MockSmartSwitchChassis() @@ -2014,6 +2079,141 @@ def is_valid_date(date_str): assert is_valid_date(chassis_state_db[key]["dpu_midplane_link_time"]) + +def _make_smartswitch_updater_with_dpu(name="DPU0"): + """Helper: build a SmartSwitchModuleUpdater with a single DPU module.""" + chassis = MockSmartSwitchChassis() + module = MockModule(0, name, "DPU Module 0", ModuleBase.MODULE_TYPE_DPU, 0, "{}-0000".format(name)) + module.set_midplane_ip() + chassis.module_list.append(module) + module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) + module_updater.midplane_initialized = True + return module_updater, module + + +@pytest.fixture(autouse=True) +def _isolate_midplane_reason_dir(tmp_path): + """Redirect persisted midplane-down-reason files to a per-test temp dir.""" + import chassisd + original = chassisd.MODULE_REBOOT_CAUSE_DIR + chassisd.MODULE_REBOOT_CAUSE_DIR = str(tmp_path) + yield str(tmp_path) + chassisd.MODULE_REBOOT_CAUSE_DIR = original + + +@pytest.mark.parametrize("platform_reason, expected", [ + # (major, "") -> only the major part is rendered + ((ChassisBase.REBOOT_CAUSE_THERMAL_OVERLOAD_ASIC, ""), + "Unplanned: 'Thermal Overload: ASIC'"), + # (major, minor) -> both parts are rendered + ((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "kernel panic"), + "Unplanned: 'Hardware - Other, kernel panic'"), + # falsy-but-valid minor (0) is kept; guards against `if minor` truthiness, + # only None/"" should omit the minor part. + ((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, 0), + "Unplanned: 'Hardware - Other, 0'"), +]) +def test_resolve_midplane_down_reason_unplanned(platform_reason, expected): + """Unplanned down: platform reason tuple is rendered as Unplanned: ''.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason(platform_reason) + + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + assert reason == expected + + +def test_resolve_midplane_down_reason_unplanned_unknown(): + """Unplanned down: no platform reason falls back to Unknown.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason(None) + + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + assert reason == "Unplanned: 'Unknown'" + + +def test_resolve_midplane_down_reason_planned(): + """Planned down: transition flag set -> Planned: ''.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.set_module_state_transition("DPU0", "shutdown") + + module_updater.state_db.hget = MagicMock(return_value="shutdown") + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + + assert reason == "Planned: 'shutdown'" + + +def test_midplane_down_reason_persisted_in_db(): + """check_midplane_reachability records the midplane-down reason in CHASSIS_STATE_DB.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_THERMAL_OVERLOAD_OTHER, "")) + + chassis_state_db = {} + + def mock_hset(key, field, value): + chassis_state_db.setdefault(key, {})[field] = value + + def mock_hget(key, field): + return chassis_state_db.get(key, {}).get(field) + + with patch.object(module_updater, 'chassis_state_db') as mock_db: + mock_db.hset = MagicMock(side_effect=mock_hset) + mock_db.hget = MagicMock(side_effect=mock_hget) + + # midplane up first, then down + module.set_midplane_reachable(True) + module_updater.check_midplane_reachability() + key = "DPU_STATE|DPU0" + assert chassis_state_db[key]["dpu_midplane_link_state"] == "up" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "" + + module.set_midplane_reachable(False) + module_updater.check_midplane_reachability() + assert chassis_state_db[key]["dpu_midplane_link_state"] == "down" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "Unplanned: 'Thermal Overload: Other'" + + +def test_midplane_down_reason_persisted_to_file_and_cleared(): + """Full lifecycle: down persists the reason to file, restart reads it back, up clears it.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_THERMAL_OVERLOAD_ASIC, "")) + path = module_updater._midplane_reason_path("DPU0") + + chassis_state_db = {} + + def mock_hset(key, field, value): + chassis_state_db.setdefault(key, {})[field] = value + + def mock_hget(key, field): + return chassis_state_db.get(key, {}).get(field) + + with patch.object(module_updater, 'chassis_state_db') as mock_db: + mock_db.hset = MagicMock(side_effect=mock_hset) + mock_db.hget = MagicMock(side_effect=mock_hget) + + module.set_midplane_reachable(False) + module_updater.check_midplane_reachability() + key = "DPU_STATE|DPU0" + with open(path) as f: + assert f.read().strip() == "Unplanned: 'Thermal Overload: ASIC'" + assert chassis_state_db[key]["dpu_midplane_link_state"] == "down" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "Unplanned: 'Thermal Overload: ASIC'" + + restarted_updater, restarted_module = _make_smartswitch_updater_with_dpu() + restarted_module.clear_module_state_transition("DPU0") + restarted_module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "boom")) + resolved = restarted_updater._resolve_midplane_down_reason(restarted_module, "DPU0") + assert resolved == "Unplanned: 'Thermal Overload: ASIC'" + + # Up: persisted reason file removed. + module.set_midplane_reachable(True) + module_updater.check_midplane_reachability() + assert not os.path.exists(path) + + def test_submit_dpu_callback(): """Test that submit_dpu_callback calls the right functions in the correct order""" chassis = MockSmartSwitchChassis() diff --git a/sonic-chassisd/tests/test_dpu_auto_recovery.py b/sonic-chassisd/tests/test_dpu_auto_recovery.py index 12536d50a..356751b6e 100644 --- a/sonic-chassisd/tests/test_dpu_auto_recovery.py +++ b/sonic-chassisd/tests/test_dpu_auto_recovery.py @@ -2021,46 +2021,6 @@ def test_set_last_ready_time_format(self): class TestRebootCausePersistence: """Test reboot cause file I/O, symlink management, and history rotation.""" - def test_persist_dpu_reboot_time(self): - """persist_dpu_reboot_time writes formatted time to file.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - updater.persist_dpu_reboot_time("DPU0") - - path = os.path.join(tmpdir, "dpu0", "prev_reboot_time.txt") - assert os.path.exists(path) - content = open(path).read().strip() - # Format: YYYY_MM_DD_HH_MM_SS - assert len(content.split('_')) == 6 - - def test_retrieve_dpu_reboot_time_exists(self): - """retrieve_dpu_reboot_time returns stored time.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "prev_reboot_time.txt"), 'w') as f: - f.write("2026_05_19_10_30_00") - - result = updater.retrieve_dpu_reboot_time("DPU0") - assert result == "2026_05_19_10_30_00" - - def test_retrieve_dpu_reboot_time_missing(self): - """retrieve_dpu_reboot_time returns None when file doesn't exist.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - result = updater.retrieve_dpu_reboot_time("DPU0") - assert result is None - def test_persist_dpu_reboot_cause_creates_history_file(self): """persist_dpu_reboot_cause creates JSON history file.""" chassis = create_chassis_with_dpus(1) @@ -2183,35 +2143,6 @@ def test_rotate_files_no_op_when_under_limit(self): updater._rotate_files("DPU0") assert len(os.listdir(history_dir)) == 3 - def test_retrieve_dpu_reboot_info_valid(self): - """retrieve_dpu_reboot_info returns (cause, time) from JSON file.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - data = {"cause": "Kernel Panic", "name": "2026_05_19_10_00_00"} - with open(os.path.join(mod_dir, "previous-reboot-cause.json"), 'w') as f: - json.dump(data, f) - - cause, time_str = updater.retrieve_dpu_reboot_info("DPU0") - assert cause == "Kernel Panic" - assert time_str == "2026_05_19_10_00_00" - - def test_retrieve_dpu_reboot_info_missing_file(self): - """retrieve_dpu_reboot_info returns (None, None) when file doesn't exist.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - cause, time_str = updater.retrieve_dpu_reboot_info("DPU0") - assert cause is None - assert time_str is None - - # ============================================================================ # Test: update_dpu_reboot_cause_to_db # ============================================================================ @@ -2408,48 +2339,6 @@ def test_cascading_dp_down_then_cp_down(self): assert updater.dpu_recovery_state["DPU0"]['state'] == DPU_STATE_WAIT_FOR_SELF_RECOVERY -# ============================================================================ -# Test: _is_first_boot helper -# ============================================================================ - -class TestIsFirstBoot: - """Test _is_first_boot() helper method.""" - - def test_first_boot_detected(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "reboot-cause.txt"), 'w') as f: - f.write("First boot") - - assert updater._is_first_boot("DPU0") is True - - def test_not_first_boot(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "reboot-cause.txt"), 'w') as f: - f.write("Watchdog") - - assert updater._is_first_boot("DPU0") is False - - def test_missing_file_returns_false(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - assert updater._is_first_boot("DPU0") is False - - # ============================================================================ # Test: Exception paths and edge cases for coverage # ============================================================================ diff --git a/sonic-chassisd/tests/test_dpu_chassisd.py b/sonic-chassisd/tests/test_dpu_chassisd.py index bf7dd3eed..3a8af0d56 100644 --- a/sonic-chassisd/tests/test_dpu_chassisd.py +++ b/sonic-chassisd/tests/test_dpu_chassisd.py @@ -28,11 +28,19 @@ def load_source(module_name, module_path): SYSLOG_IDENTIFIER = 'dpu_chassisd_test' +TEST_BOOT_ID = 'test-boot-id' daemon_base.db_connect = MagicMock() test_path = os.path.dirname(os.path.abspath(__file__)) os.environ["CHASSISD_UNIT_TESTING"] = "1" +@pytest.fixture(autouse=True) +def mock_dpu_boot_id(): + """Return a deterministic boot ID instead of reading the test host.""" + with mock.patch('chassisd.DpuStateUpdater.get_boot_id', return_value=TEST_BOOT_ID): + yield + + @pytest.mark.parametrize('conf_db, app_db, expected_state', [ ({'Ethernet0': {}}, {'Ethernet0': [True, 'up']}, 'up'), ({'Ethernet0': {}}, {'Ethernet0': [True, 'down']}, 'down'), @@ -90,13 +98,16 @@ def test_dpu_state_update_api(state, expected_state): @pytest.mark.parametrize('dpu_id, dp_state, cp_state, expected_state', [ (0, False, False, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, False, True, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, True, True, {'DPU0': {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), ]) def test_dpu_state_update(dpu_id, dp_state, cp_state, expected_state): chassis = MockDpuChassis() @@ -127,19 +138,23 @@ def hset(key, field, value): # After the deinit we assume that the DPU state is down. assert chassis_state_db == {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}} @pytest.mark.parametrize('dpu_id, dp_state, cp_state, expected_state', [ (0, False, False, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, False, True, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, True, True, {'DPU0': {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), ]) def test_dpu_state_manager(dpu_id, dp_state, cp_state, expected_state): chassis = MockDpuChassis() @@ -173,7 +188,8 @@ def hset(key, field, value): # After the deinit we assume that the DPU state is down. assert chassis_state_db == {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}} def test_dpu_chassis_daemon(): @@ -208,18 +224,30 @@ def hset(key, field, value): # Wait for thread to start and update DB time.sleep(3) - assert chassis_state_db == {'DPU1': - {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + assert chassis_state_db == { + 'DPU1': { + 'dpu_data_plane_state': 'up', + 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'dpu_control_plane_state': 'up', + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID, + } + } daemon_chassisd.signal_handler(signal.SIGINT, None) daemon_chassisd.stop.wait.return_value = True thread.join() - assert chassis_state_db == {'DPU1': - {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + assert chassis_state_db == { + 'DPU1': { + 'dpu_data_plane_state': 'down', + 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'dpu_control_plane_state': 'down', + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID, + } + } with mock.patch.object(swsscommon.Table, 'hset', side_effect=hset): daemon_chassisd = DpuChassisdDaemon(SYSLOG_IDENTIFIER, chassis) daemon_chassisd.CHASSIS_INFO_UPDATE_PERIOD_SECS = MagicMock(return_value=1) @@ -277,7 +305,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} @@ -313,7 +342,8 @@ def mock_pop(): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} @@ -348,7 +378,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} # Verify current states are tracked @@ -374,9 +405,12 @@ def hset(key, field, value): chassis_state_db[key][field] = value # Test with same state update + event = ('DPU0', 'SET', (('dpu_data_plane_state', 'up'), + ('dpu_control_plane_state', 'up'), + ('boot_id', TEST_BOOT_ID))) with mock.patch.object(swsscommon.Table, 'hset', side_effect=hset): with mock.patch.object(swsscommon.SubscriberStateTable, 'pop', - return_value=('DPU0', 'SET', (('dpu_data_plane_state', 'up'), ('dpu_control_plane_state', 'up')))): + return_value=event): with mock.patch.object(swsscommon.Select, 'select', side_effect=[(swsscommon.Select.OBJECT, None), (swsscommon.Select.OBJECT, None), KeyboardInterrupt]): @@ -388,8 +422,8 @@ def hset(key, field, value): dpu_state_mng.current_cp_state = 'up' dpu_state_mng.task_worker() - # Verify no updates occurred since states were unchanged - assert update_count == 0 + # Only the initial boot ID publication is expected; the duplicate event adds no update. + assert update_count == 1 def test_dpu_state_manager_different_dpu(): @@ -424,8 +458,8 @@ def hset(key, field, value): dpu_state_mng.current_cp_state = 'up' dpu_state_mng.task_worker() - # Verify no updates occurred since it was for a different DPU - assert update_count == 0 + # Only the initial boot ID publication is expected; the other DPU event is ignored. + assert update_count == 1 def test_dpu_state_manager_state_change(): @@ -462,7 +496,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'down', # Changed to down - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} # Verify current states were updated