diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 33fa9f89875..72f6c256878 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -5,6 +5,7 @@ REBOOT_USER="$(logname 2> /dev/null || whoami)" REBOOT_TIME=$(date) REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt" WARM_DIR=/host/warmboot +FLOW_CNT_TRAP_MARKER="${WARM_DIR}/flow_cnt_trap_need_re_enable" REDIS_FILE=dump.rdb CONFIG_DB_FILE=/etc/sonic/config_db.json REBOOT_SCRIPT_NAME=$(basename $0) @@ -365,6 +366,14 @@ function clear_boot() if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true fi + + # roll back FLOW_CNT_TRAP=disable if we disabled it but never reached + # kexec. Marker presence proves we owned the disable. + if [[ -f "$FLOW_CNT_TRAP_MARKER" ]]; then + debug "Restoring FLOW_CNT_TRAP=enable after aborted warm-reboot" + execute_in_namespaces asic enable_flow_cnt_trap || /bin/true + rm -f "$FLOW_CNT_TRAP_MARKER" + fi } function init_warm_reboot_states() @@ -1118,11 +1127,62 @@ function pause_orchagent() debug "Orchagent paused successfully" } +# When FLOW_CNT_TRAP is enabled, orchagent binds a +# SAI_COUNTER per HOSTIF trap via SAI_HOSTIF_TRAP_ATTR_COUNTER_ID. After warm +# boot, APPLY_VIEW replays that attribute and Broadcom SAI fails the lookup: +# _brcm_sai_set_hostif_entryctr: field entry stat attach failed with error +# Entry not found (BCM_E_NOT_FOUND) -> SAI_STATUS_ITEM_NOT_FOUND +# syncd then enters shutdown-wait and orchagent crash-loops. Disabling the +# flex counter pre-shutdown drives CoppOrch::clearHostIfTrapCounterIdList(), +# which unbinds every counter while the SAI handle is still valid -- no +# replay is needed on the new boot. The marker file signals +# finalize-warmboot.sh to re-enable the counter after reconcile. +function disable_flow_cnt_trap_for_warmboot() +{ + local key="FLEX_COUNTER_TABLE|FLOW_CNT_TRAP" + local cur + cur=$(sonic-db-cli -n "$NETNS" CONFIG_DB HGET "$key" FLEX_COUNTER_STATUS 2>/dev/null) + if [[ "$cur" != "enable" ]]; then + debug "FLOW_CNT_TRAP not enabled (status='${cur}'), skipping pre-warmboot disable" + # Drop any marker left by a prior aborted warm-reboot so it doesn't + # falsely re-enable a counter the user has since disabled. + rm -f "$FLOW_CNT_TRAP_MARKER" + return + fi + debug "FLOW_CNT_TRAP=enable: disabling pre-warmboot" + mkdir -p "${WARM_DIR}" + touch "$FLOW_CNT_TRAP_MARKER" + sonic-db-cli -n "$NETNS" CONFIG_DB HSET "$key" FLEX_COUNTER_STATUS disable > /dev/null + # Poll COUNTERS_TRAP_NAME_MAP until CoppOrch drains it — that proves every + # SAI_HOSTIF_TRAP_ATTR_COUNTER_ID unbind has completed in orchagent's thread. + local deadline=$((SECONDS + 10)) + local n + while (( SECONDS < deadline )); do + n=$(sonic-db-cli -n "$NETNS" COUNTERS_DB HLEN COUNTERS_TRAP_NAME_MAP 2>/dev/null || echo 0) + [[ "$n" == "0" ]] && break + done + if [[ "${n:-0}" != "0" ]]; then + debug "WARN: COUNTERS_TRAP_NAME_MAP still has ${n} entries after 10s; proceeding" + fi +} + +# Re-enable FLOW_CNT_TRAP in CONFIG_DB. Used by clear_boot to roll back the +# disable_flow_cnt_trap_for_warmboot CONFIG_DB write when warm-reboot aborts +# before kexec. +function enable_flow_cnt_trap() +{ + sonic-db-cli -n "$NETNS" CONFIG_DB HSET "FLEX_COUNTER_TABLE|FLOW_CNT_TRAP" FLEX_COUNTER_STATUS enable > /dev/null +} + # After orchagent pause, there's no rollback if anything goes wrong on Multi-ASIC devices, force execution if [[ $NUM_ASIC -gt 1 ]]; then FORCE=yes fi +if [[ "$REBOOT_TYPE" = "warm-reboot" ]]; then + execute_in_namespaces asic disable_flow_cnt_trap_for_warmboot +fi + if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "express-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then execute_in_namespaces asic pause_orchagent fi