diff --git a/sonic-chassisd/scripts/chassisd b/sonic-chassisd/scripts/chassisd index deb409c38..91db46d9c 100755 --- a/sonic-chassisd/scripts/chassisd +++ b/sonic-chassisd/scripts/chassisd @@ -81,7 +81,6 @@ CHASSIS_MODULE_REBOOT_TIMESTAMP_FIELD = 'timestamp' CHASSIS_MODULE_REBOOT_REBOOT_FIELD = 'reboot' DEFAULT_LINECARD_REBOOT_TIMEOUT = 180 DEFAULT_DPU_REBOOT_TIMEOUT = 360 -MAX_DPU_REBOOT_DURATION = 800 PLATFORM_ENV_CONF_FILE = "/usr/share/sonic/platform/platform_env.conf" PLATFORM_JSON_FILE = "/usr/share/sonic/platform/platform.json" @@ -162,6 +161,9 @@ DPU_STATE_MANUAL_INTERVENTION = 'ManualIntervention' DPU_STATE_ADMIN_DOWN = 'AdminDown' DPU_STATE_UNRECOVERABLE = 'Unrecoverable' +# DPU boot_id fields +BOOT_ID = 'boot_id' +BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id' class DpuStates(NamedTuple): """Container for DPU plane states to avoid positional confusion.""" @@ -695,7 +697,7 @@ class ModuleUpdater(logger.Logger): # Chassis app db cleanup of all asics of the module # Get the module key and host name from down_modules key - module, lc = re.split('\|', module_host) + module, lc = re.split(r'\|', module_host) if lc == '': # Host name is not available for this module. No clean up is needed @@ -756,10 +758,10 @@ class SmartSwitchModuleUpdater(ModuleUpdater): self.chassis = chassis self.num_modules = self.chassis.get_num_modules() # Connect to STATE_DB and create chassis info tables - state_db = daemon_base.db_connect("STATE_DB") - self.chassis_table = swsscommon.Table(state_db, CHASSIS_INFO_TABLE) - self.module_table = swsscommon.Table(state_db, CHASSIS_MODULE_INFO_TABLE) - self.midplane_table = swsscommon.Table(state_db, CHASSIS_MIDPLANE_INFO_TABLE) + self.state_db = daemon_base.db_connect("STATE_DB") + self.chassis_table = swsscommon.Table(self.state_db, CHASSIS_INFO_TABLE) + self.module_table = swsscommon.Table(self.state_db, CHASSIS_MODULE_INFO_TABLE) + self.midplane_table = swsscommon.Table(self.state_db, CHASSIS_MIDPLANE_INFO_TABLE) self.info_dict_keys = [CHASSIS_MODULE_INFO_NAME_FIELD, CHASSIS_MODULE_INFO_DESC_FIELD, CHASSIS_MODULE_INFO_SLOT_FIELD, @@ -844,24 +846,81 @@ class SmartSwitchModuleUpdater(ModuleUpdater): else: return ModuleBase.MODULE_STATUS_EMPTY - def retrieve_dpu_reboot_info(self, module): + def retrieve_dpu_reboot_info(self, module_name): """ - Retrieve the most recent reboot cause and time from previous-reboot-cause.json. - Returns (cause_string, time_string), or (None, None) if unavailable. + Retrieve the most recent reboot cause, time and boot_id from + previous-reboot-cause.json. + Returns (cause_string, time_string, boot_id), or (None, None, None) + if unavailable. """ try: - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "previous-reboot-cause.json") + path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module_name.lower(), "previous-reboot-cause.json") if os.path.exists(path): with open(path, 'r') as f: data = json.load(f) cause = data.get("cause") time_str = data.get("name") # Format: "YYYY_MM_DD_HH_MM_SS" - return cause, time_str + boot_id = data.get("boot_id") + return cause, time_str, boot_id else: - self.log_debug(f"{module}: previous-reboot-cause.json not found") + self.log_debug(f"{module_name}: previous-reboot-cause.json not found") except Exception as e: - self.log_error(f"{module}: Failed to read previous-reboot-cause.json: {e}") - return None, None + self.log_error(f"{module_name}: Failed to read previous-reboot-cause.json: {e}") + return None, None, None + + def dpu_boot_id_update(self, module_name, current_boot_id): + """ + Update the reboot cause and boot_id for a single DPU when new boot_id is detected. + + Args: + module_name: DPU name, e.g. "DPU0" (the DPU_STATE row key). + current_boot_id: boot_id reported for that DPU in CHASSIS_STATE_DB. + """ + # No boot_id -> nothing to capture. + if not current_boot_id: + return + # Get the previous boot_id from the previous-reboot-cause.json file. + _, _, previous_boot_id = self.retrieve_dpu_reboot_info(module_name) + # Same boot_id as the one already recorded -> nothing to capture. + if previous_boot_id == current_boot_id: + return + # Get the module from module_name. + try: + module_index = try_get(self.chassis.get_module_index, module_name, default=INVALID_MODULE_INDEX) + # get_module(-1) would return the last module instead of failing + if module_index < 0: + self.log_error(f"Unable to get module-index for {module_name} to capture reboot cause") + return + module = self.chassis.get_module(module_index) + except Exception as e: + self.log_error(f"Failed to look up module {module_name} to capture reboot cause: {e}") + return + if module is None: + self.log_error(f"No module object for {module_name} to capture reboot cause") + return + self.log_notice(f"{module_name}: new boot_id {current_boot_id} detected, capturing reboot cause") + + # Get the reboot cause from the module. + try: + reboot_cause = try_get(module.get_reboot_cause) + except Exception as e: + self.log_error(f"Failed to get reboot cause for {module_name}: {e}") + return + + # Persist the reboot cause and boot_id to the file. + try: + self.persist_dpu_reboot_cause(reboot_cause, module_name, boot_id=current_boot_id) + except Exception as e: + self.log_error(f"Failed to persist reboot cause for {module_name}: {e}") + return + + # Update the reboot cause to the DB. + try: + self.update_dpu_reboot_cause_to_db(module_name) + except Exception as e: + self.log_error(f"Failed to update reboot cause to DB for {module_name}: {e}. " + "The boot_id and reboot cause is stored in json file.") + return def module_db_update(self): for module_index in range(0, self.num_modules): @@ -870,63 +929,15 @@ class SmartSwitchModuleUpdater(ModuleUpdater): key = module_info_dict[CHASSIS_MODULE_INFO_NAME_FIELD] if not key.startswith(ModuleBase.MODULE_TYPE_DPU): - self.log_error("Incorrect module-name {}. Should start with {} ".format(key, - ModuleBase.MODULE_TYPE_DPU)) + self.log_error(f"Incorrect module-name {key}. Should start with {ModuleBase.MODULE_TYPE_DPU} ") continue fvs = swsscommon.FieldValuePairs([(CHASSIS_MODULE_INFO_DESC_FIELD, module_info_dict[CHASSIS_MODULE_INFO_DESC_FIELD]), (CHASSIS_MODULE_INFO_SLOT_FIELD, module_info_dict[CHASSIS_MODULE_INFO_SLOT_FIELD]), (CHASSIS_MODULE_INFO_OPERSTATUS_FIELD, module_info_dict[CHASSIS_MODULE_INFO_OPERSTATUS_FIELD]), (CHASSIS_MODULE_INFO_SERIAL_FIELD, module_info_dict[CHASSIS_MODULE_INFO_SERIAL_FIELD])]) - - # Get a copy of the previous operational status of the module - prev_status = self.get_module_current_status(key) self.module_table.set(key, fvs) - # Get a copy of the current operational status of the module - current_status = module_info_dict[CHASSIS_MODULE_INFO_OPERSTATUS_FIELD] - - # Operational status transitioning to offline - if prev_status != ModuleBase.MODULE_STATUS_EMPTY and prev_status != str(ModuleBase.MODULE_STATUS_OFFLINE) and current_status == str(ModuleBase.MODULE_STATUS_OFFLINE): - self.log_notice("{} operational status transitioning to offline".format(key)) - - # Persist dpu down time - self.persist_dpu_reboot_time(key) - # persist reboot cause - reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause) - self.persist_dpu_reboot_cause(reboot_cause, key) - # publish reboot cause to db - self.update_dpu_reboot_cause_to_db(key) - - elif (prev_status == ModuleBase.MODULE_STATUS_EMPTY or prev_status == str(ModuleBase.MODULE_STATUS_OFFLINE)) and current_status != str(ModuleBase.MODULE_STATUS_OFFLINE): - self.log_notice(f"{key} operational status transitioning to online") - - reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause) - if isinstance(reboot_cause, (tuple, list)): - current_cause = reboot_cause[0] - else: - current_cause = reboot_cause - - stored_cause, stored_time_str = self.retrieve_dpu_reboot_info(key) - - is_reboot = False - if current_cause and stored_cause and stored_time_str: - try: - stored_dt = datetime.strptime(stored_time_str, "%Y_%m_%d_%H_%M_%S").replace(tzinfo=timezone.utc) - now = datetime.now(timezone.utc) - delta_sec = (now - stored_dt).total_seconds() - - if current_cause == stored_cause and delta_sec < MAX_DPU_REBOOT_DURATION: - self.log_info(f"{key}: is_reboot=True — same reboot cause within {int(delta_sec)}s") - is_reboot = True - except Exception as e: - self.log_error(f"{key}: Reboot cause/time comparison failed: {e}") - - if not is_reboot and (stored_time_str is not None or self._is_first_boot(key)): - # persist reboot cause and publish to db - self.persist_dpu_reboot_cause(reboot_cause, key) - self.update_dpu_reboot_cause_to_db(key) - def _get_module_info(self, module_index): """ Retrieves module info of this module @@ -950,10 +961,79 @@ class SmartSwitchModuleUpdater(ModuleUpdater): return module_info_dict - def update_dpu_state(self, key, state): + def _resolve_midplane_down_reason(self, module, module_name): + """ + Build the dpu_midplane_link_reason for an up->down midplane transition. + + Planned : a graceful admin operation set the transition flag, so the + 'transition_type' field is read from STATE_DB + CHASSIS_MODULE_TABLE and reported as Planned: ''. + Unplanned : no transition flag is set, so the platform is queried via + get_midplane_down_reason() and reported as Unplanned: ''. + """ + # If the reason is stored in a file, return it. + cached = self._read_midplane_down_reason(module_name) + if cached: + return cached + + reason_str = None + # Planned operation (graceful shutdown/startup/reset) in progress? + if try_get(module.get_module_state_transition, module_name, default=False): + try: + module_key = "{}|{}".format(CHASSIS_MODULE_INFO_TABLE, module_name.upper()) + transition_type = self.state_db.hget(module_key, "transition_type") + if transition_type: + reason_str = "Planned: '{}'".format(transition_type) + except Exception as e: + self.log_error("{}: failed to read transition_type: {}".format(module_name, e)) + + if reason_str is None: + # Unplanned down: ask the platform for the hardware reason. + reason = try_get(module.get_midplane_down_reason, default=None) + if isinstance(reason, (tuple, list)): + parts = [str(p) for p in reason if p not in (None, "")] + detail = ", ".join(parts) if parts else "Unknown" + else: + detail = reason or "Unknown" + reason_str = "Unplanned: '{}'".format(detail) + + self._write_midplane_down_reason(module_name, reason_str) + return reason_str + + def _midplane_reason_path(self, module_name): + return os.path.join(MODULE_REBOOT_CAUSE_DIR, module_name.lower(), "midplane-down-reason.txt") + + def _read_midplane_down_reason(self, module_name): + try: + with open(self._midplane_reason_path(module_name)) as f: + return f.read().strip() or None + except FileNotFoundError: + return None + except Exception as e: + self.log_error("{}: read midplane reason failed: {}".format(module_name, e)) + return None + + def _write_midplane_down_reason(self, module_name, reason): + path = self._midplane_reason_path(module_name) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + self._atomic_write_text(path, (reason or "") + "\n") + except Exception as e: + self.log_error("{}: persist midplane reason failed: {}".format(module_name, e)) + + def _clear_midplane_down_reason(self, module_name): + try: + os.remove(self._midplane_reason_path(module_name)) + except FileNotFoundError: + pass + except Exception as e: + self.log_error("{}: clear midplane reason failed: {}".format(module_name, e)) + + def update_dpu_state(self, key, state, reason=None): """ Update specific DPU state fields in chassisStateDB using the given key. - If state is 'down', set control plane, data plane states to down as well. + If state is 'down', set control plane, data plane states to down as well + and record the midplane-down reason. """ try: # Connect to the CHASSIS_STATE_DB using daemon_base @@ -961,10 +1041,10 @@ class SmartSwitchModuleUpdater(ModuleUpdater): self.chassis_state_db = daemon_base.db_connect("CHASSIS_STATE_DB") - # Prepare the fields to update + # Prepare the fields to update. Coerce reason to "" so the 'up' + # path (reason=None) never writes a None into the DB. updates = { - "dpu_midplane_link_state": state, - "dpu_midplane_link_reason": "", + "dpu_midplane_link_reason": reason or "", "dpu_midplane_link_time": get_formatted_time(), } # If midplane state is down, set control plane, data plane states to down as well @@ -972,6 +1052,9 @@ class SmartSwitchModuleUpdater(ModuleUpdater): updates[CP_STATE] = "down" updates[DP_STATE] = "down" + # Write the state last so a partial update is retried on the next poll. + updates["dpu_midplane_link_state"] = state + # Update each field directly for field, value in updates.items(): self.chassis_state_db.hset(key, field, value) @@ -1011,38 +1094,45 @@ class SmartSwitchModuleUpdater(ModuleUpdater): """Generates the full path for history files.""" return os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "history", file_name) - def _is_first_boot(self, module): - """Checks if the reboot-cause file indicates a first boot.""" - file_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "reboot-cause.txt") - + @staticmethod + def _atomic_write(file_path, write_fn): + """Let write_fn fill a temporary file in the same directory, then rename it into place.""" + tmp_path = "{}.tmp".format(file_path) try: - with open(file_path, 'r') as f: - content = f.read().strip() - return content == "First boot" - except FileNotFoundError: - return False - - def persist_dpu_reboot_time(self, module): - """Persist the current reboot time to a file.""" - time_str = self._get_current_time_str() - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write(time_str) - - def retrieve_dpu_reboot_time(self, module): - """Retrieve the persisted reboot time from a file.""" - path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - + with open(tmp_path, 'w') as f: + write_fn(f) + os.replace(tmp_path, file_path) + except Exception: + if os.path.lexists(tmp_path): + os.remove(tmp_path) + raise + + @classmethod + def _atomic_write_json(cls, file_path, content): + """Write content to file_path as JSON, atomically.""" + cls._atomic_write(file_path, lambda f: json.dump(content, f)) + + @classmethod + def _atomic_write_text(cls, file_path, text): + """Write text to file_path atomically.""" + cls._atomic_write(file_path, lambda f: f.write(text)) + + @staticmethod + def _atomic_replace_symlink(target_path, symlink_path): + """Point symlink_path at target_path without ever leaving symlink_path absent.""" + tmp_path = "{}.tmp".format(symlink_path) + if os.path.lexists(tmp_path): + os.remove(tmp_path) + os.symlink(target_path, tmp_path) try: - with open(path, 'r') as f: - return f.read().strip() - except FileNotFoundError: - return None + os.replace(tmp_path, symlink_path) + except Exception: + if os.path.lexists(tmp_path): + os.remove(tmp_path) + raise - def persist_dpu_reboot_cause(self, reboot_cause, module): - """Persist the reboot cause information and handle file rotation.""" + def persist_dpu_reboot_cause(self, reboot_cause, module, boot_id=None): + """Persist the reboot cause information, boot_id and handle file rotation.""" # Extract cause and comment from the reboot_cause if reboot_cause: try: @@ -1055,19 +1145,11 @@ class SmartSwitchModuleUpdater(ModuleUpdater): else: cause, comment = "Unknown", "N/A" - prev_reboot_time = self.retrieve_dpu_reboot_time(module) - if prev_reboot_time is None: - prev_reboot_time = self._get_current_time_str() - - file_name = f"{prev_reboot_time}_reboot_cause.json" - prev_reboot_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "prev_reboot_time.txt") - - if os.path.exists(prev_reboot_path): - os.remove(prev_reboot_path) - + reboot_time = self._get_current_time_str() + file_name = f"{reboot_time}_reboot_cause.json" file_path = self._get_history_path(module, file_name) try: - formatted_time = get_formatted_time(datetimeobj=datetime.strptime(prev_reboot_time, "%Y_%m_%d_%H_%M_%S")) + formatted_time = get_formatted_time(datetimeobj=datetime.strptime(reboot_time, "%Y_%m_%d_%H_%M_%S")) except ValueError: formatted_time = get_formatted_time() @@ -1076,24 +1158,14 @@ class SmartSwitchModuleUpdater(ModuleUpdater): "comment": comment, "device": module, "time": formatted_time, - "name": prev_reboot_time, + "name": reboot_time, + "boot_id": boot_id or "", } - with open(file_path, 'w') as f: - json.dump(reboot_cause_dict, f) - - # Write the reboot_cause content to the reboot-cause.txt file, overwriting it - reboot_cause_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "reboot-cause.txt") - os.makedirs(os.path.dirname(reboot_cause_path), exist_ok=True) - with open(reboot_cause_path, 'w') as cause_file: - cause_file.write(json.dumps(reboot_cause) + '\n') - - # Update symlink to the latest reboot cause file + # write reboot_cause_dict to the file and replace the symlink to the latest reboot cause file. + self._atomic_write_json(file_path, reboot_cause_dict) symlink_path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "previous-reboot-cause.json") - if os.path.exists(symlink_path): - os.remove(symlink_path) - if os.path.exists(file_path): - os.symlink(file_path, symlink_path) + self._atomic_replace_symlink(file_path, symlink_path) # Perform file rotation if necessary self._rotate_files(module) @@ -1103,7 +1175,7 @@ class SmartSwitchModuleUpdater(ModuleUpdater): history_dir = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "history") os.makedirs(history_dir, exist_ok=True) try: - files = sorted(os.listdir(history_dir)) + files = sorted(f for f in os.listdir(history_dir) if f.endswith("_reboot_cause.json")) except FileNotFoundError: return @@ -1188,10 +1260,13 @@ class SmartSwitchModuleUpdater(ModuleUpdater): # Update midplane state in the chassisStateDB DPU_STATE table key = DPU_STATE_TABLE + "|" + module_key dpu_mp_state = self.get_dpu_midplane_state(key) - if midplane_access and dpu_mp_state != 'up': - self.update_dpu_state(key, 'up') + if midplane_access: + if dpu_mp_state != 'up': + self.update_dpu_state(key, "up") + self._clear_midplane_down_reason(module_key) elif not midplane_access and dpu_mp_state != 'down': - self.update_dpu_state(key, "down") + midplane_down_reason = self._resolve_midplane_down_reason(module, module_key) + self.update_dpu_state(key, "down", midplane_down_reason) # Update db with midplane information fvs = swsscommon.FieldValuePairs([(CHASSIS_MIDPLANE_INFO_IP_FIELD, midplane_ip), @@ -1713,6 +1788,57 @@ class SmartSwitchConfigManagerTask(ProcessTaskBase): self.config_updater.module_config_update(key, admin_state) + +# +# Reboot-cause subscriber task ============================================= +# + + +class RebootCauseSubscriberTask(ProcessTaskBase): + """ Capture DPU reboot-cause on a boot_id change. """ + + def __init__(self): + super().__init__() + self.logger = logger.Logger(SYSLOG_IDENTIFIER) + + def task_worker(self): + # Construct the updater after ProcessTaskBase forks so none of its platform or + # database objects are inherited from the parent process. + self.module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, get_chassis()) + self.chassis_state_db = daemon_base.db_connect("CHASSIS_STATE_DB") + + sel = swsscommon.Select() + sst = swsscommon.SubscriberStateTable(self.chassis_state_db, "DPU_STATE") + sel.addSelectable(sst) + + # Listen indefinitely for DPU_STATE changes in CHASSIS_STATE_DB + try: + while True: + # Use timeout so SIGTERM (handled in the parent) can stop us + (state, c) = sel.select(SELECT_TIMEOUT) + + if state == swsscommon.Select.TIMEOUT: + continue + if state != swsscommon.Select.OBJECT: + self.logger.log_warning("sel.select() did not return swsscommon.Select.OBJECT") + continue + + result = sst.pop() + if result is None: + continue + (key, op, fvp) = result + if op != "SET" or fvp is None: + continue + + fvp_dict = dict(fvp) + # check if the boot_id has changed for the DPU + if BOOT_ID in fvp_dict: + self.module_updater.dpu_boot_id_update(key, fvp_dict[BOOT_ID]) + + except KeyboardInterrupt: + pass + + # # State Manager task ======================================================== # @@ -1780,14 +1906,24 @@ class DpuStateUpdater(logger.Logger): self.dpu_state_table.hset(self.name, CP_STATE, state) self.dpu_state_table.hset(self.name, CP_UPDATE_TIME, self._time_now()) + def _update_boot_id(self, boot_id): + self.dpu_state_table.hset(self.name, BOOT_ID, boot_id) + def get_dp_state(self): return 'up' if self._get_dp_state() else 'down' def get_cp_state(self): return 'up' if self._get_cp_state() else 'down' - def update_state(self): + def get_boot_id(self): + try: + with open(BOOT_ID_PATH) as f: + return f.read().strip() + except OSError as err: + self.log_warning(f"Failed to read boot_id from {BOOT_ID_PATH}: {err}") + return None + def update_state(self): dp_current_state = self.get_dp_state() _, dp_prev_state = self.dpu_state_table.hget(self.name, DP_STATE) @@ -1799,7 +1935,14 @@ class DpuStateUpdater(logger.Logger): if cp_current_state != cp_prev_state: self._update_cp_dpu_state(cp_current_state) - return [dp_current_state, cp_current_state] + + current_boot_id = self.get_boot_id() + if current_boot_id: + _, prev_boot_id = self.dpu_state_table.hget(self.name, BOOT_ID) + if current_boot_id != prev_boot_id: + self._update_boot_id(current_boot_id) + + return [dp_current_state, cp_current_state, current_boot_id] def deinit(self): self._update_dp_dpu_state('down') @@ -1874,7 +2017,13 @@ class ChassisdDaemon(daemon_base.DaemonBase): op_state = 'up' else: op_state = 'down' - self.module_updater.update_dpu_state(dpu_state_key, op_state) + persisted_reason = None + if op_state == 'down': + persisted_reason = self.module_updater._read_midplane_down_reason(module_name) + if persisted_reason: + self.module_updater.update_dpu_state(dpu_state_key, op_state, persisted_reason) + else: + self.module_updater.update_dpu_state(dpu_state_key, op_state) if op is not None: # Create and start a thread for the DPU logic @@ -1912,18 +2061,21 @@ class ChassisdDaemon(daemon_base.DaemonBase): self.log_error("Chassisd not supported for this platform") sys.exit(CHASSIS_NOT_SUPPORTED) + self.config_manager = None + self.reboot_cause_subscriber = None + try: - # Start configuration manager task + # Start configuration manager and reboot cause subscriber tasks if self.smartswitch: self.set_initial_dpu_admin_state() self.module_updater.init_dpu_recovery_state() self.config_manager = SmartSwitchConfigManagerTask() self.config_manager.task_run() + self.reboot_cause_subscriber = RebootCauseSubscriberTask() + self.reboot_cause_subscriber.task_run() elif self.module_updater.supervisor_slot == self.module_updater.my_slot: self.config_manager = ConfigManagerTask() self.config_manager.task_run() - else: - self.config_manager = None # Start main loop self.log_info("Start daemon main loop") @@ -1943,6 +2095,8 @@ class ChassisdDaemon(daemon_base.DaemonBase): # https://github.com/sonic-net/sonic-buildimage/issues/24775 if self.config_manager is not None: self.config_manager.task_stop() + if self.reboot_cause_subscriber is not None: + self.reboot_cause_subscriber.task_stop() # Delete all the information from DB and then exit self.module_updater.deinit() @@ -1962,7 +2116,8 @@ class DpuStateManagerTask(ProcessTaskBase): self.chassis_state_db = daemon_base.db_connect('CHASSIS_STATE_DB') self.current_dp_state = None self.current_cp_state = None - + self.current_boot_id = self.dpu_state_updater.get_boot_id() + def task_worker(self): sel = swsscommon.Select() selectable = [ @@ -1974,6 +2129,10 @@ class DpuStateManagerTask(ProcessTaskBase): for s in selectable: sel.addSelectable(s) + # write boot_id into DPU_STATE table before entering the loop + if self.current_boot_id: + self.dpu_state_updater._update_boot_id(self.current_boot_id) + try: while True: (state, c) = sel.select(SELECT_TIMEOUT) @@ -2002,7 +2161,8 @@ class DpuStateManagerTask(ProcessTaskBase): fvs = dict(fvp) # No need to update if the state is the same as the current state if ('dpu_data_plane_state' in fvs and fvs['dpu_data_plane_state'] == self.current_dp_state) and \ - ('dpu_control_plane_state' in fvs and fvs['dpu_control_plane_state'] == self.current_cp_state): + ('dpu_control_plane_state' in fvs and fvs['dpu_control_plane_state'] == self.current_cp_state) and \ + ('boot_id' in fvs and fvs['boot_id'] == self.current_boot_id): update_required = False continue self.logger.log_info(f"DPU_STATE change detected: operation={op}, key={key}") @@ -2012,7 +2172,7 @@ class DpuStateManagerTask(ProcessTaskBase): break if update_required: - [self.current_dp_state, self.current_cp_state] = self.dpu_state_updater.update_state() + [self.current_dp_state, self.current_cp_state, self.current_boot_id] = self.dpu_state_updater.update_state() except KeyboardInterrupt: pass diff --git a/sonic-chassisd/tests/mock_platform.py b/sonic-chassisd/tests/mock_platform.py index fb157f21d..90b684e5e 100644 --- a/sonic-chassisd/tests/mock_platform.py +++ b/sonic-chassisd/tests/mock_platform.py @@ -34,6 +34,8 @@ def __init__(self, module_index, module_name, module_desc, module_type, module_s self.admin_state = 1 self.supervisor_slot = 16 self.midplane_access = False + self.state_transition = False + self.state_transition_type = None self.asic_list = asic_list self.module_serial = module_serial @@ -84,15 +86,18 @@ def set_admin_state_gracefully(self, up): def clear_module_state_transition(self, module_name): """Mock implementation of clear_module_state_transition""" + self.state_transition = False return True def set_module_state_transition(self, module_name, transition_type): """Mock implementation of set_module_state_transition""" + self.state_transition = True + self.state_transition_type = transition_type return True def get_module_state_transition(self, module_name): """Mock implementation of get_module_state_transition""" - return False + return self.state_transition def clear_module_gnoi_halt_in_progress(self): """Mock implementation of clear_module_gnoi_halt_in_progress""" @@ -112,6 +117,12 @@ def is_midplane_reachable(self): def set_midplane_reachable(self, up): self.midplane_access = up + def get_midplane_down_reason(self): + return getattr(self, 'midplane_down_reason', None) + + def set_midplane_down_reason(self, reason): + self.midplane_down_reason = reason + def get_all_asics(self): return self.asic_list diff --git a/sonic-chassisd/tests/test_chassisd.py b/sonic-chassisd/tests/test_chassisd.py index 81c1555b1..de6462cdd 100644 --- a/sonic-chassisd/tests/test_chassisd.py +++ b/sonic-chassisd/tests/test_chassisd.py @@ -5,11 +5,25 @@ import json import pytest import time +import importlib.util +import importlib.machinery + from mock import Mock, MagicMock, patch, mock_open from sonic_py_common import daemon_base +from sonic_platform_base.chassis_base import ChassisBase from .mock_platform import MockChassis, MockSmartSwitchChassis, MockModule from .mock_module_base import ModuleBase + +# imp is deprecated in Python 3.12 +def load_source(module_name, file_path): + loader = importlib.machinery.SourceFileLoader(module_name, file_path) + spec = importlib.util.spec_from_file_location(module_name, file_path, loader=loader) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module # required: `from chassisd import *` relies on this + loader.exec_module(module) + return module + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../scripts")) # Assuming OBJECT should be a specific value, define it manually @@ -221,65 +235,170 @@ def test_smartswitch_moduleupdater_status_transitions(): # Create the updater module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) - # Mock dependent methods - with patch.object(module_updater, 'retrieve_dpu_reboot_info', return_value=("Switch rebooted DPU", "2023_01_01_00_00_00")) as mock_reboot_info, \ - patch.object(module_updater, '_is_first_boot', return_value=False) as mock_is_first_boot, \ - patch.object(module_updater, 'persist_dpu_reboot_cause') as mock_persist_reboot_cause, \ - patch.object(module_updater, 'update_dpu_reboot_cause_to_db') as mock_update_reboot_db, \ - patch("os.makedirs") as mock_makedirs, \ - patch("builtins.open", mock_open()) as mock_file, \ - patch.object(module_updater, '_get_history_path', return_value="/tmp/prev_reboot_time.txt") as mock_get_history_path: - - # Transition from ONLINE to OFFLINE - offline_status = ModuleBase.MODULE_STATUS_OFFLINE - module.set_oper_status(offline_status) - module_updater.module_db_update() - assert module.get_oper_status() == offline_status - - # Reset mocks for next transition - mock_file.reset_mock() - mock_makedirs.reset_mock() - mock_persist_reboot_cause.reset_mock() - mock_update_reboot_db.reset_mock() + # Transition from ONLINE to OFFLINE + offline_status = ModuleBase.MODULE_STATUS_OFFLINE + module.set_oper_status(offline_status) + module_updater.module_db_update() + assert module.get_oper_status() == offline_status - # Ensure ONLINE transition is handled correctly - online_status = ModuleBase.MODULE_STATUS_ONLINE - module.set_oper_status(online_status) - module_updater.module_db_update() - assert module.get_oper_status() == online_status + # Ensure ONLINE transition is handled correctly + online_status = ModuleBase.MODULE_STATUS_ONLINE + module.set_oper_status(online_status) + module_updater.module_db_update() + assert module.get_oper_status() == online_status - # Validate mock calls for ONLINE transition - mock_persist_reboot_cause.assert_called_once() - mock_update_reboot_db.assert_called_once() -def test_online_transition_skips_reboot_update(): +def _make_boot_id_updater(): + """Helper: build a SmartSwitchModuleUpdater with one DPU for boot_id consumer tests.""" chassis = MockSmartSwitchChassis() - index = 0 - name = "DPU0" - module = MockModule(index, name, "DPU", ModuleBase.MODULE_TYPE_DPU, 0, "SN123") - module.set_oper_status(ModuleBase.MODULE_STATUS_OFFLINE) + module = MockModule(0, "DPU0", "DPU Module 0", ModuleBase.MODULE_TYPE_DPU, 0, "DPU0-0000") chassis.module_list.append(module) - updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) + return updater + + +def _patch_persisted_boot_id(updater, boot_id): + """Make the persisted record report boot_id as the baseline.""" + return patch.object(updater, 'retrieve_dpu_reboot_info', + return_value=("Kernel Panic", "2026_05_19_10_00_00", boot_id)) + + +def test_dpu_boot_id_update_new_boot(): + """New boot_id -> reboot cause captured.""" + updater = _make_boot_id_updater() + + with _patch_persisted_boot_id(updater, "old-boot-id"), \ + patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.dpu_boot_id_update("DPU0", "new-boot-id") + + mock_persist.assert_called_once() + # boot_id must be forwarded to persist so it lands in the json/db. + assert mock_persist.call_args.kwargs.get("boot_id") == "new-boot-id" + mock_update_db.assert_called_once_with("DPU0") + + +def test_dpu_boot_id_update_same_boot(): + """Unchanged boot_id -> nothing captured (avoids duplicate on every event).""" + updater = _make_boot_id_updater() + + with _patch_persisted_boot_id(updater, "same-boot-id"), \ + patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.dpu_boot_id_update("DPU0", "same-boot-id") + + mock_persist.assert_not_called() + mock_update_db.assert_not_called() + + +@pytest.mark.parametrize("boot_id", [None, ""]) +def test_dpu_boot_id_update_no_boot_id(boot_id): + """Empty/None boot_id -> nothing captured.""" + updater = _make_boot_id_updater() + + with _patch_persisted_boot_id(updater, "old-boot-id"), \ + patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.dpu_boot_id_update("DPU0", boot_id) + + mock_persist.assert_not_called() + mock_update_db.assert_not_called() + + +def test_dpu_boot_id_update_db_failure_keeps_record(tmp_path): + """A failed DB refresh is logged, and the record it persisted still becomes the baseline. + + retrieve_dpu_reboot_info is left unmocked and the record is written to a real directory, so + the second event has to read the file back from disk. Feeding the baseline in from a mock + would assert nothing about whether the DB failure cost us the record. + """ + updater = _make_boot_id_updater() + history_dir = tmp_path / "dpu0" / "history" + history_dir.mkdir(parents=True) + + with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", str(tmp_path)), \ + patch.object(updater, 'update_dpu_reboot_cause_to_db', side_effect=Exception("db down")), \ + patch.object(updater, 'log_error') as mock_log_error: + updater.dpu_boot_id_update("DPU0", "new-boot-id") + + assert mock_log_error.called + records = list(history_dir.glob("*_reboot_cause.json")) + assert len(records) == 1 + assert json.loads(records[0].read_text())["boot_id"] == "new-boot-id" + + # The same boot_id again: the record on disk is now the baseline, so nothing is re-captured + # even though the DB never received the first one. Those rows are restored by the next + # capture or at NPU boot. + with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", str(tmp_path)), \ + patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist: + updater.dpu_boot_id_update("DPU0", "new-boot-id") + + mock_persist.assert_not_called() - # Mock the module going ONLINE - module.set_oper_status(ModuleBase.MODULE_STATUS_ONLINE) - with patch.object(updater, 'retrieve_dpu_reboot_info', - return_value=("Switch rebooted DPU", datetime.now(timezone.utc).strftime("%Y_%m_%d_%H_%M_%S"))), \ - patch.object(module, 'get_reboot_cause', return_value="Switch rebooted DPU"), \ - patch.object(updater, '_is_first_boot', return_value=False), \ +def test_dpu_boot_id_update_unknown_module(): + """Unknown DPU name (no module index) -> nothing captured.""" + updater = _make_boot_id_updater() + + with _patch_persisted_boot_id(updater, "old-boot-id"), \ patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ - patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update, \ - patch("builtins.open", mock_open()), \ - patch("os.makedirs"), \ - patch.object(updater, '_get_history_path', return_value="/tmp/fake.json"): + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db: + updater.dpu_boot_id_update("DPU_NONEXISTENT", "new-boot-id") - updater.module_db_update() + mock_persist.assert_not_called() + mock_update_db.assert_not_called() + + +@pytest.mark.parametrize("lookup, expected_log", [ + ({"target": "get_module_index", "side_effect": KeyError("DPU0")}, "Failed to look up module"), + ({"target": "get_module_index", "side_effect": RuntimeError("platform not ready")}, "Failed to look up module"), + ({"target": "get_module", "side_effect": IndexError("out of range")}, "Failed to look up module"), + ({"target": "get_module", "return_value": None}, "No module object"), +]) +def test_dpu_boot_id_update_module_lookup_failure_is_contained(lookup, expected_log): + """A platform lookup that raises or yields no module is logged, never propagated. + + The subscriber loop that calls this has no exception boundary, so an escaping exception + would silently end DPU reboot-cause capture for the rest of the daemon's lifetime. + + A platform that returns no module must be reported as such: without an explicit check it + surfaces one step later as an AttributeError blamed on reading the reboot cause. + """ + updater = _make_boot_id_updater() + behavior = dict(lookup) + target = behavior.pop("target") + + with _patch_persisted_boot_id(updater, "old-boot-id"), \ + patch.object(updater.chassis, target, **behavior), \ + patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db, \ + patch.object(updater, 'log_error') as mock_log_error: + updater.dpu_boot_id_update("DPU0", "new-boot-id") - # Ensure no reboot update due to is_reboot = True mock_persist.assert_not_called() - mock_update.assert_not_called() + mock_update_db.assert_not_called() + assert expected_log in mock_log_error.call_args.args[0] + + +@pytest.mark.parametrize("failing_call, expected_log", [ + ("module.get_reboot_cause", "Failed to get reboot cause"), + ("updater.persist_dpu_reboot_cause", "Failed to persist reboot cause"), +]) +def test_dpu_boot_id_update_capture_failure_is_contained(failing_call, expected_log): + """A failing capture step is logged and abandons the capture, leaving the DB untouched.""" + updater = _make_boot_id_updater() + owner_name, attr = failing_call.split(".") + owner = updater if owner_name == "updater" else updater.chassis.get_module(0) + + with _patch_persisted_boot_id(updater, "old-boot-id"), \ + patch.object(owner, attr, side_effect=Exception("boom")), \ + patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update_db, \ + patch.object(updater, 'log_error') as mock_log_error: + updater.dpu_boot_id_update("DPU0", "new-boot-id") + + mock_update_db.assert_not_called() + assert expected_log in mock_log_error.call_args.args[0] + def test_retrieve_dpu_reboot_info_success(): class DummyChassis: @@ -287,12 +406,13 @@ def get_num_modules(self): return 0 def init_midplane_switch(self): return False updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis()) - sample_json = {"cause": "Switch rebooted DPU", "name": "2025_06_25_17_18_52"} + sample_json = {"cause": "Switch rebooted DPU", "name": "2025_06_25_17_18_52", "boot_id": "e4252288-be0d-40ec-8338-d1e5ec206771"} with patch("os.path.exists", return_value=True), \ patch("builtins.open", mock_open(read_data=json.dumps(sample_json))): - cause, time_str = updater.retrieve_dpu_reboot_info("dpu0") + cause, time_str, boot_id = updater.retrieve_dpu_reboot_info("dpu0") assert cause == "Switch rebooted DPU" assert time_str == "2025_06_25_17_18_52" + assert boot_id == "e4252288-be0d-40ec-8338-d1e5ec206771" def test_retrieve_dpu_reboot_info_file_missing(): class DummyChassis: @@ -301,9 +421,110 @@ def init_midplane_switch(self): return False # required for SmartSwitchModuleUp updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis()) with patch("os.path.exists", return_value=False): - cause, time_str = updater.retrieve_dpu_reboot_info("dpu0") + cause, time_str, boot_id = updater.retrieve_dpu_reboot_info("dpu0") assert cause is None assert time_str is None + assert boot_id is None + + +def test_reboot_cause_subscriber_processes_boot_id(): + """Subscriber creates its updater in the child and forwards a valid boot_id event.""" + module_updater = MagicMock(spec=SmartSwitchModuleUpdater) + chassis = MagicMock() + subscriber = RebootCauseSubscriberTask() + subscriber_db = MagicMock() + mock_select = MagicMock() + mock_sst = MagicMock() + select_object = swsscommon.Select.OBJECT + select_timeout = swsscommon.Select.TIMEOUT + + mock_select.select.side_effect = [(select_object, None), KeyboardInterrupt] + mock_sst.pop.return_value = ("DPU0", "SET", (("boot_id", "new-boot-id"),)) + + with patch("chassisd.get_chassis", return_value=chassis) as mock_get_chassis, \ + patch("chassisd.SmartSwitchModuleUpdater", return_value=module_updater) as mock_updater_class, \ + patch("chassisd.daemon_base.db_connect", return_value=subscriber_db) as mock_db_connect, \ + patch("chassisd.swsscommon.Select", return_value=mock_select) as mock_select_class, \ + patch("chassisd.swsscommon.SubscriberStateTable", return_value=mock_sst) as mock_sst_class: + mock_select_class.TIMEOUT = select_timeout + mock_select_class.OBJECT = select_object + subscriber.task_worker() + + mock_get_chassis.assert_called_once_with() + mock_updater_class.assert_called_once_with(SYSLOG_IDENTIFIER, chassis) + mock_db_connect.assert_called_once_with("CHASSIS_STATE_DB") + mock_sst_class.assert_called_once_with(subscriber_db, "DPU_STATE") + mock_select.addSelectable.assert_called_once_with(mock_sst) + module_updater.dpu_boot_id_update.assert_called_once_with("DPU0", "new-boot-id") + +def test_atomic_write_json_replaces_content(tmp_path): + """The final path holds the new content and no temporary file is left behind.""" + target = tmp_path / "record.json" + target.write_text('{"cause": "old"}') + + SmartSwitchModuleUpdater._atomic_write_json(str(target), {"cause": "new"}) + + assert json.loads(target.read_text()) == {"cause": "new"} + assert list(p.name for p in tmp_path.iterdir()) == ["record.json"] + + +def test_atomic_write_json_keeps_previous_content_on_failure(tmp_path): + """A failed write leaves the previous record intact and removes the temporary file.""" + target = tmp_path / "record.json" + target.write_text('{"cause": "old"}') + + with patch("builtins.open", side_effect=OSError("disk full")): + with pytest.raises(OSError): + SmartSwitchModuleUpdater._atomic_write_json(str(target), {"cause": "new"}) + + assert json.loads(target.read_text()) == {"cause": "old"} + assert not (tmp_path / "record.json.tmp").exists() + + +def test_atomic_replace_symlink_never_leaves_link_absent(tmp_path): + """Replacing the link repoints it in one step instead of removing and recreating it.""" + old_record = tmp_path / "old_reboot_cause.json" + new_record = tmp_path / "new_reboot_cause.json" + old_record.write_text("{}") + new_record.write_text("{}") + link = tmp_path / "previous-reboot-cause.json" + os.symlink(str(old_record), str(link)) + + removed = [] + real_remove = os.remove + + def tracking_remove(path): + removed.append(path) + real_remove(path) + + with patch("chassisd.os.remove", side_effect=tracking_remove): + SmartSwitchModuleUpdater._atomic_replace_symlink(str(new_record), str(link)) + + assert os.path.realpath(str(link)) == os.path.realpath(str(new_record)) + # Removing the live link, even briefly, would lose the persisted baseline on a crash. + assert str(link) not in removed + + +def test_get_boot_id_reads_kernel_boot_id(): + """get_boot_id returns the stripped kernel boot ID.""" + updater = DpuStateUpdater.__new__(DpuStateUpdater) + updater._syslog = MagicMock() + + with patch("builtins.open", mock_open(read_data="test-boot-id\n")): + assert updater.get_boot_id() == "test-boot-id" + + +def test_get_boot_id_returns_none_on_oserror(): + """get_boot_id returns None and logs a warning when the file cannot be read.""" + updater = DpuStateUpdater.__new__(DpuStateUpdater) + updater._syslog = MagicMock() + updater.log_warning = MagicMock() + + with patch("builtins.open", side_effect=OSError("boot ID unavailable")): + assert updater.get_boot_id() is None + + updater.log_warning.assert_called_once() + def test_smartswitch_moduleupdater_check_invalid_name(): chassis = MockSmartSwitchChassis() @@ -679,40 +900,6 @@ def test_update_dpu_reboot_cause_to_db(mock_open, mock_glob): mock_log_warning.assert_any_call("Error processing file /host/reboot-cause/module/dpu0/history/file1.txt: Unable to read file") -def test_smartswitch_module_db_update(): - chassis = MockSmartSwitchChassis() - reboot_cause = "Power loss" - key = "DPU0" - index = 0 - name = "DPU0" - desc = "DPU Module 0" - slot = 0 - serial = "DPU0-0000" - module_type = ModuleBase.MODULE_TYPE_DPU - module = MockModule(index, name, desc, module_type, slot, serial) - - # Set initial state - status = ModuleBase.MODULE_STATUS_ONLINE - module.set_oper_status(status) - chassis.module_list.append(module) - - module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) - expected_path = "/host/reboot-cause/module/reboot_cause/dpu0/history/2024_11_13_15_06_40_reboot_cause.txt" - symlink_path = "/host/reboot-cause/module/dpu0/previous-reboot-cause.json" - - with patch("os.path.exists", return_value=True), \ - patch("os.makedirs") as mock_makedirs, \ - patch("builtins.open", mock_open(read_data="Power loss")) as mock_file, \ - patch("os.remove") as mock_remove, \ - patch("os.symlink") as mock_symlink: - - # Call the function to test - module_updater.persist_dpu_reboot_cause(reboot_cause, key) - module_updater._is_first_boot(name) - module_updater.persist_dpu_reboot_time(name) - module_updater.update_dpu_reboot_cause_to_db(name) - - def test_platform_json_file_exists_and_valid(): """Test case where the platform JSON file exists with valid data.""" chassis = MockSmartSwitchChassis() @@ -1499,7 +1686,7 @@ def test_set_initial_dpu_admin_state_up(): mock_submit_callback.assert_not_called() -def test_set_initial_dpu_admin_state_empty_offline(): +def test_set_initial_dpu_admin_state_empty_offline(midplane_reason_dir): """Test set_initial_dpu_admin_state when admin state is empty and operational state is offline""" chassis = MockSmartSwitchChassis() @@ -1529,6 +1716,10 @@ def test_set_initial_dpu_admin_state_empty_offline(): daemon_chassisd.platform_chassis = chassis daemon_chassisd.smartswitch = True + reason_dir = midplane_reason_dir / "dpu0" + reason_dir.mkdir() + (reason_dir / "midplane-down-reason.txt").write_text("Unplanned: 'Thermal Overload: ASIC'\n") + # Mock the necessary methods - admin state is EMPTY, operational state is OFFLINE with patch.object(module_updater, 'get_module_admin_status', return_value=ModuleBase.MODULE_STATUS_EMPTY), \ patch.object(module_updater, 'update_dpu_state') as mock_update_dpu_state, \ @@ -1543,8 +1734,9 @@ def test_set_initial_dpu_admin_state_empty_offline(): mock_clear_transition.assert_called_once() mock_clear_gnoi.assert_called_once() - # Verify DPU state was updated with 'down' since operational state is OFFLINE - mock_update_dpu_state.assert_called_once_with("DPU_STATE|DPU0", 'down') + # Verify the persisted reason is restored with the down state. + mock_update_dpu_state.assert_called_once_with( + "DPU_STATE|DPU0", 'down', "Unplanned: 'Thermal Overload: ASIC'") # Verify callback was submitted with MODULE_ADMIN_DOWN when admin state is EMPTY mock_submit_callback.assert_called_once_with(0, MODULE_ADMIN_DOWN) @@ -1582,6 +1774,7 @@ def test_set_initial_dpu_admin_state_empty_not_offline(): # Mock the necessary methods - admin state is EMPTY, operational state is PRESENT with patch.object(module_updater, 'get_module_admin_status', return_value=ModuleBase.MODULE_STATUS_EMPTY), \ + patch.object(module_updater, '_read_midplane_down_reason', return_value=None), \ patch.object(module_updater, 'update_dpu_state') as mock_update_dpu_state, \ patch.object(daemon_chassisd, 'submit_dpu_callback') as mock_submit_callback, \ patch.object(module, 'clear_module_state_transition') as mock_clear_transition, \ @@ -1950,7 +2143,7 @@ def is_valid_date(date_str): AssertionError("Date is not set!") assert is_valid_date(date_value) -def test_smartswitch_moduleupdater_midplane_state_change(): +def test_smartswitch_moduleupdater_midplane_state_change(midplane_reason_dir): """Test that when midplane goes down, control plane and data plane states are set to down""" chassis = MockSmartSwitchChassis() index = 0 @@ -1990,6 +2183,12 @@ def mock_hget(key, field): # Verify initial state key = "DPU_STATE|" + name assert chassis_state_db[key]["dpu_midplane_link_state"] == "up" + chassis_state_db[key].update({ + CP_UPDATE_TIME: "original-cp-time", + "dpu_control_plane_reason": "original-cp-reason", + DP_UPDATE_TIME: "original-dp-time", + "dpu_data_plane_reason": "original-dp-reason", + }) # Now set midplane as down module.set_midplane_reachable(False) @@ -1999,6 +2198,10 @@ def mock_hget(key, field): assert chassis_state_db[key]["dpu_midplane_link_state"] == "down" assert chassis_state_db[key]["dpu_control_plane_state"] == "down" assert chassis_state_db[key]["dpu_data_plane_state"] == "down" + assert chassis_state_db[key][CP_UPDATE_TIME] == "original-cp-time" + assert chassis_state_db[key]["dpu_control_plane_reason"] == "original-cp-reason" + assert chassis_state_db[key][DP_UPDATE_TIME] == "original-dp-time" + assert chassis_state_db[key]["dpu_data_plane_reason"] == "original-dp-reason" # Verify timestamps are set assert "dpu_midplane_link_time" in chassis_state_db[key] @@ -2014,6 +2217,162 @@ def is_valid_date(date_str): assert is_valid_date(chassis_state_db[key]["dpu_midplane_link_time"]) + +def _make_smartswitch_updater_with_dpu(name="DPU0"): + """Helper: build a SmartSwitchModuleUpdater with a single DPU module.""" + chassis = MockSmartSwitchChassis() + module = MockModule(0, name, "DPU Module 0", ModuleBase.MODULE_TYPE_DPU, 0, "{}-0000".format(name)) + module.set_midplane_ip() + chassis.module_list.append(module) + module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis) + module_updater.midplane_initialized = True + return module_updater, module + + +@pytest.fixture +def midplane_reason_dir(tmp_path, monkeypatch): + """Redirect persisted midplane-down-reason files to a per-test temp dir.""" + import chassisd + monkeypatch.setattr(chassisd, "MODULE_REBOOT_CAUSE_DIR", str(tmp_path)) + return tmp_path + + +@pytest.mark.parametrize("platform_reason, expected", [ + # (major, "") -> only the major part is rendered + ((ChassisBase.REBOOT_CAUSE_THERMAL_OVERLOAD_ASIC, ""), + "Unplanned: 'Thermal Overload: ASIC'"), + # (major, minor) -> both parts are rendered + ((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "kernel panic"), + "Unplanned: 'Hardware - Other, kernel panic'"), + # falsy-but-valid minor (0) is kept; guards against `if minor` truthiness, + # only None/"" should omit the minor part. + ((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, 0), + "Unplanned: 'Hardware - Other, 0'"), +]) +def test_resolve_midplane_down_reason_unplanned(platform_reason, expected, midplane_reason_dir): + """Unplanned down: platform reason tuple is rendered as Unplanned: ''.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason(platform_reason) + + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + assert reason == expected + + +def test_resolve_midplane_down_reason_unplanned_unknown(midplane_reason_dir): + """Unplanned down: no platform reason falls back to Unknown.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason(None) + + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + assert reason == "Unplanned: 'Unknown'" + + +def test_resolve_midplane_down_reason_planned(midplane_reason_dir): + """Planned down: transition flag set -> Planned: ''.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.set_module_state_transition("DPU0", "shutdown") + + module_updater.state_db.hget = MagicMock(return_value="shutdown") + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + + assert reason == "Planned: 'shutdown'" + + +def test_resolve_midplane_down_reason_missing_transition_type(midplane_reason_dir): + """A disappearing transition type must not produce Planned: 'unknown'.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.set_module_state_transition("DPU0", "shutdown") + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "link failure")) + module_updater.state_db.hget = MagicMock(return_value=None) + + reason = module_updater._resolve_midplane_down_reason(module, "DPU0") + + assert reason == "Unplanned: 'Hardware - Other, link failure'" + + +def test_midplane_down_state_retried_after_partial_db_failure(midplane_reason_dir): + """A partial DB write leaves state unchanged so the next poll retries the full update.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "link failure")) + module.set_midplane_reachable(False) + key = "DPU_STATE|DPU0" + chassis_state_db = {key: {"dpu_midplane_link_state": "up"}} + fail_cp_write = [True] + + def mock_hset(db_key, field, value): + if field == CP_STATE and fail_cp_write[0]: + fail_cp_write[0] = False + raise RuntimeError("DB write failed") + chassis_state_db.setdefault(db_key, {})[field] = value + + def mock_hget(db_key, field): + return chassis_state_db.get(db_key, {}).get(field) + + with patch.object(module_updater, 'chassis_state_db') as mock_db: + mock_db.hset = MagicMock(side_effect=mock_hset) + mock_db.hget = MagicMock(side_effect=mock_hget) + + module_updater.check_midplane_reachability() + assert chassis_state_db[key]["dpu_midplane_link_state"] == "up" + + module_updater.check_midplane_reachability() + assert chassis_state_db[key]["dpu_midplane_link_state"] == "down" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "Unplanned: 'Hardware - Other, link failure'" + assert chassis_state_db[key][CP_STATE] == "down" + assert chassis_state_db[key][DP_STATE] == "down" + + +def test_midplane_down_reason_persisted_to_file_and_cleared(midplane_reason_dir): + """Full lifecycle: down persists the reason to file, restart reads it back, up clears it.""" + module_updater, module = _make_smartswitch_updater_with_dpu() + module.clear_module_state_transition("DPU0") + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_THERMAL_OVERLOAD_ASIC, "")) + path = module_updater._midplane_reason_path("DPU0") + + chassis_state_db = {} + + def mock_hset(key, field, value): + chassis_state_db.setdefault(key, {})[field] = value + + def mock_hget(key, field): + return chassis_state_db.get(key, {}).get(field) + + with patch.object(module_updater, 'chassis_state_db') as mock_db: + mock_db.hset = MagicMock(side_effect=mock_hset) + mock_db.hget = MagicMock(side_effect=mock_hget) + + module.set_midplane_reachable(False) + module_updater.check_midplane_reachability() + key = "DPU_STATE|DPU0" + with open(path) as f: + assert f.read().strip() == "Unplanned: 'Thermal Overload: ASIC'" + assert chassis_state_db[key]["dpu_midplane_link_state"] == "down" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "Unplanned: 'Thermal Overload: ASIC'" + + # Repeated down polls keep the first reason even if the live reason changes. + module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "boom")) + module_updater.check_midplane_reachability() + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "Unplanned: 'Thermal Overload: ASIC'" + with open(path) as f: + assert f.read().strip() == "Unplanned: 'Thermal Overload: ASIC'" + + restarted_updater, restarted_module = _make_smartswitch_updater_with_dpu() + restarted_module.clear_module_state_transition("DPU0") + restarted_module.set_midplane_down_reason((ChassisBase.REBOOT_CAUSE_HARDWARE_OTHER, "boom")) + resolved = restarted_updater._resolve_midplane_down_reason(restarted_module, "DPU0") + assert resolved == "Unplanned: 'Thermal Overload: ASIC'" + + # Up: persisted reason file removed. + module.set_midplane_reachable(True) + module_updater.check_midplane_reachability() + assert not os.path.exists(path) + assert chassis_state_db[key]["dpu_midplane_link_state"] == "up" + assert chassis_state_db[key]["dpu_midplane_link_reason"] == "" + + def test_submit_dpu_callback(): """Test that submit_dpu_callback calls the right functions in the correct order""" chassis = MockSmartSwitchChassis() diff --git a/sonic-chassisd/tests/test_dpu_auto_recovery.py b/sonic-chassisd/tests/test_dpu_auto_recovery.py index 12536d50a..ada61cbd6 100644 --- a/sonic-chassisd/tests/test_dpu_auto_recovery.py +++ b/sonic-chassisd/tests/test_dpu_auto_recovery.py @@ -2021,46 +2021,6 @@ def test_set_last_ready_time_format(self): class TestRebootCausePersistence: """Test reboot cause file I/O, symlink management, and history rotation.""" - def test_persist_dpu_reboot_time(self): - """persist_dpu_reboot_time writes formatted time to file.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - updater.persist_dpu_reboot_time("DPU0") - - path = os.path.join(tmpdir, "dpu0", "prev_reboot_time.txt") - assert os.path.exists(path) - content = open(path).read().strip() - # Format: YYYY_MM_DD_HH_MM_SS - assert len(content.split('_')) == 6 - - def test_retrieve_dpu_reboot_time_exists(self): - """retrieve_dpu_reboot_time returns stored time.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "prev_reboot_time.txt"), 'w') as f: - f.write("2026_05_19_10_30_00") - - result = updater.retrieve_dpu_reboot_time("DPU0") - assert result == "2026_05_19_10_30_00" - - def test_retrieve_dpu_reboot_time_missing(self): - """retrieve_dpu_reboot_time returns None when file doesn't exist.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - result = updater.retrieve_dpu_reboot_time("DPU0") - assert result is None - def test_persist_dpu_reboot_cause_creates_history_file(self): """persist_dpu_reboot_cause creates JSON history file.""" chassis = create_chassis_with_dpus(1) @@ -2141,6 +2101,29 @@ def test_persist_dpu_reboot_cause_creates_symlink(self): target = os.readlink(symlink) assert "_reboot_cause.json" in target + def test_persist_dpu_reboot_cause_write_failure_keeps_previous_baseline(self): + """A failed record write leaves the previous baseline intact and no partial file behind.""" + chassis = create_chassis_with_dpus(1) + updater = create_updater(chassis) + + with tempfile.TemporaryDirectory() as tmpdir: + with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): + history_dir = os.path.join(tmpdir, "dpu0", "history") + os.makedirs(history_dir) + symlink = os.path.join(tmpdir, "dpu0", "previous-reboot-cause.json") + + updater._get_current_time_str = MagicMock(return_value="2026_01_01_00_00_00") + updater.persist_dpu_reboot_cause(("First", ""), "DPU0", boot_id="boot-1") + + updater._get_current_time_str = MagicMock(return_value="2026_01_01_00_00_01") + with patch("chassisd.json.dump", side_effect=OSError("disk full")): + with pytest.raises(OSError): + updater.persist_dpu_reboot_cause(("Second", ""), "DPU0", boot_id="boot-2") + + # Baseline still resolves to the first record, so the second boot is re-captured later. + assert os.readlink(symlink).endswith("2026_01_01_00_00_00_reboot_cause.json") + assert os.listdir(history_dir) == ["2026_01_01_00_00_00_reboot_cause.json"] + def test_rotate_files_removes_old_files(self): """_rotate_files removes oldest files when exceeding MAX_HISTORY_FILES.""" chassis = create_chassis_with_dpus(1) @@ -2183,35 +2166,6 @@ def test_rotate_files_no_op_when_under_limit(self): updater._rotate_files("DPU0") assert len(os.listdir(history_dir)) == 3 - def test_retrieve_dpu_reboot_info_valid(self): - """retrieve_dpu_reboot_info returns (cause, time) from JSON file.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - data = {"cause": "Kernel Panic", "name": "2026_05_19_10_00_00"} - with open(os.path.join(mod_dir, "previous-reboot-cause.json"), 'w') as f: - json.dump(data, f) - - cause, time_str = updater.retrieve_dpu_reboot_info("DPU0") - assert cause == "Kernel Panic" - assert time_str == "2026_05_19_10_00_00" - - def test_retrieve_dpu_reboot_info_missing_file(self): - """retrieve_dpu_reboot_info returns (None, None) when file doesn't exist.""" - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - cause, time_str = updater.retrieve_dpu_reboot_info("DPU0") - assert cause is None - assert time_str is None - - # ============================================================================ # Test: update_dpu_reboot_cause_to_db # ============================================================================ @@ -2408,48 +2362,6 @@ def test_cascading_dp_down_then_cp_down(self): assert updater.dpu_recovery_state["DPU0"]['state'] == DPU_STATE_WAIT_FOR_SELF_RECOVERY -# ============================================================================ -# Test: _is_first_boot helper -# ============================================================================ - -class TestIsFirstBoot: - """Test _is_first_boot() helper method.""" - - def test_first_boot_detected(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "reboot-cause.txt"), 'w') as f: - f.write("First boot") - - assert updater._is_first_boot("DPU0") is True - - def test_not_first_boot(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - mod_dir = os.path.join(tmpdir, "dpu0") - os.makedirs(mod_dir) - with open(os.path.join(mod_dir, "reboot-cause.txt"), 'w') as f: - f.write("Watchdog") - - assert updater._is_first_boot("DPU0") is False - - def test_missing_file_returns_false(self): - chassis = create_chassis_with_dpus(1) - updater = create_updater(chassis) - - with tempfile.TemporaryDirectory() as tmpdir: - with patch("chassisd.MODULE_REBOOT_CAUSE_DIR", tmpdir): - assert updater._is_first_boot("DPU0") is False - - # ============================================================================ # Test: Exception paths and edge cases for coverage # ============================================================================ diff --git a/sonic-chassisd/tests/test_dpu_chassisd.py b/sonic-chassisd/tests/test_dpu_chassisd.py index bf7dd3eed..c66e5563e 100644 --- a/sonic-chassisd/tests/test_dpu_chassisd.py +++ b/sonic-chassisd/tests/test_dpu_chassisd.py @@ -28,11 +28,19 @@ def load_source(module_name, module_path): SYSLOG_IDENTIFIER = 'dpu_chassisd_test' +TEST_BOOT_ID = 'test-boot-id' daemon_base.db_connect = MagicMock() test_path = os.path.dirname(os.path.abspath(__file__)) os.environ["CHASSISD_UNIT_TESTING"] = "1" +@pytest.fixture(autouse=True) +def mock_dpu_boot_id(): + """Return a deterministic boot ID instead of reading the test host.""" + with mock.patch('chassisd.DpuStateUpdater.get_boot_id', return_value=TEST_BOOT_ID): + yield + + @pytest.mark.parametrize('conf_db, app_db, expected_state', [ ({'Ethernet0': {}}, {'Ethernet0': [True, 'up']}, 'up'), ({'Ethernet0': {}}, {'Ethernet0': [True, 'down']}, 'down'), @@ -90,13 +98,16 @@ def test_dpu_state_update_api(state, expected_state): @pytest.mark.parametrize('dpu_id, dp_state, cp_state, expected_state', [ (0, False, False, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, False, True, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, True, True, {'DPU0': {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), ]) def test_dpu_state_update(dpu_id, dp_state, cp_state, expected_state): chassis = MockDpuChassis() @@ -127,19 +138,45 @@ def hset(key, field, value): # After the deinit we assume that the DPU state is down. assert chassis_state_db == {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}} + + +def test_dpu_state_update_skips_unchanged_boot_id(): + """An already-published boot_id must not be rewritten. + + Rewriting it would hand the NPU consumer a DPU_STATE event for a boot it already captured. + Publishing a boot_id that differs from the stored one is covered by test_dpu_state_update. + """ + chassis = MockDpuChassis() + + chassis.get_dpu_id = MagicMock(return_value=0) + chassis.get_dataplane_state = MagicMock(return_value=True) + chassis.get_controlplane_state = MagicMock(return_value=True) + + dpu_updater = DpuStateUpdater(SYSLOG_IDENTIFIER, chassis) + dpu_updater._time_now = MagicMock(return_value='Sat Jan 01 12:00:00 AM UTC 2000') + dpu_updater.dpu_state_table.hset('DPU0', BOOT_ID, TEST_BOOT_ID) + + with mock.patch.object(dpu_updater, '_update_boot_id') as mock_update_boot_id: + dpu_updater.update_state() + + mock_update_boot_id.assert_not_called() @pytest.mark.parametrize('dpu_id, dp_state, cp_state, expected_state', [ (0, False, False, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, False, True, {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), (0, True, True, {'DPU0': {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}}), + 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}}), ]) def test_dpu_state_manager(dpu_id, dp_state, cp_state, expected_state): chassis = MockDpuChassis() @@ -173,7 +210,8 @@ def hset(key, field, value): # After the deinit we assume that the DPU state is down. assert chassis_state_db == {'DPU0': {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID}} def test_dpu_chassis_daemon(): @@ -208,18 +246,30 @@ def hset(key, field, value): # Wait for thread to start and update DB time.sleep(3) - assert chassis_state_db == {'DPU1': - {'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'up', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + assert chassis_state_db == { + 'DPU1': { + 'dpu_data_plane_state': 'up', + 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'dpu_control_plane_state': 'up', + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID, + } + } daemon_chassisd.signal_handler(signal.SIGINT, None) daemon_chassisd.stop.wait.return_value = True thread.join() - assert chassis_state_db == {'DPU1': - {'dpu_data_plane_state': 'down', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', - 'dpu_control_plane_state': 'down', 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000'}} + assert chassis_state_db == { + 'DPU1': { + 'dpu_data_plane_state': 'down', + 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'dpu_control_plane_state': 'down', + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID, + } + } with mock.patch.object(swsscommon.Table, 'hset', side_effect=hset): daemon_chassisd = DpuChassisdDaemon(SYSLOG_IDENTIFIER, chassis) daemon_chassisd.CHASSIS_INFO_UPDATE_PERIOD_SECS = MagicMock(return_value=1) @@ -277,7 +327,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} @@ -313,7 +364,8 @@ def mock_pop(): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} @@ -348,7 +400,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'up', - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} # Verify current states are tracked @@ -374,9 +427,12 @@ def hset(key, field, value): chassis_state_db[key][field] = value # Test with same state update + event = ('DPU0', 'SET', (('dpu_data_plane_state', 'up'), + ('dpu_control_plane_state', 'up'), + ('boot_id', TEST_BOOT_ID))) with mock.patch.object(swsscommon.Table, 'hset', side_effect=hset): with mock.patch.object(swsscommon.SubscriberStateTable, 'pop', - return_value=('DPU0', 'SET', (('dpu_data_plane_state', 'up'), ('dpu_control_plane_state', 'up')))): + return_value=event): with mock.patch.object(swsscommon.Select, 'select', side_effect=[(swsscommon.Select.OBJECT, None), (swsscommon.Select.OBJECT, None), KeyboardInterrupt]): @@ -388,8 +444,8 @@ def hset(key, field, value): dpu_state_mng.current_cp_state = 'up' dpu_state_mng.task_worker() - # Verify no updates occurred since states were unchanged - assert update_count == 0 + # Only the initial boot ID publication is expected; the duplicate event adds no update. + assert update_count == 1 def test_dpu_state_manager_different_dpu(): @@ -424,8 +480,8 @@ def hset(key, field, value): dpu_state_mng.current_cp_state = 'up' dpu_state_mng.task_worker() - # Verify no updates occurred since it was for a different DPU - assert update_count == 0 + # Only the initial boot ID publication is expected; the other DPU event is ignored. + assert update_count == 1 def test_dpu_state_manager_state_change(): @@ -462,7 +518,8 @@ def hset(key, field, value): 'dpu_data_plane_state': 'up', 'dpu_data_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', 'dpu_control_plane_state': 'down', # Changed to down - 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000' + 'dpu_control_plane_time': 'Sat Jan 01 12:00:00 AM UTC 2000', + 'boot_id': TEST_BOOT_ID }} # Verify current states were updated