Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions scripts/fast-reboot
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ REBOOT_USER="$(logname 2> /dev/null || whoami)"
REBOOT_TIME=$(date)
REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt"
WARM_DIR=/host/warmboot
FLOW_CNT_TRAP_MARKER="${WARM_DIR}/flow_cnt_trap_need_re_enable"
REDIS_FILE=dump.rdb
CONFIG_DB_FILE=/etc/sonic/config_db.json
REBOOT_SCRIPT_NAME=$(basename $0)
Expand Down Expand Up @@ -365,6 +366,14 @@ function clear_boot()
if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true
fi

# roll back FLOW_CNT_TRAP=disable if we disabled it but never reached
# kexec. Marker presence proves we owned the disable.
if [[ -f "$FLOW_CNT_TRAP_MARKER" ]]; then
debug "Restoring FLOW_CNT_TRAP=enable after aborted warm-reboot"
execute_in_namespaces asic enable_flow_cnt_trap || /bin/true
rm -f "$FLOW_CNT_TRAP_MARKER"
fi
}

function init_warm_reboot_states()
Expand Down Expand Up @@ -1118,11 +1127,62 @@ function pause_orchagent()
debug "Orchagent paused successfully"
}

# When FLOW_CNT_TRAP is enabled, orchagent binds a
# SAI_COUNTER per HOSTIF trap via SAI_HOSTIF_TRAP_ATTR_COUNTER_ID. After warm
# boot, APPLY_VIEW replays that attribute and Broadcom SAI fails the lookup:
# _brcm_sai_set_hostif_entryctr: field entry stat attach failed with error
# Entry not found (BCM_E_NOT_FOUND) -> SAI_STATUS_ITEM_NOT_FOUND
# syncd then enters shutdown-wait and orchagent crash-loops. Disabling the
# flex counter pre-shutdown drives CoppOrch::clearHostIfTrapCounterIdList(),
# which unbinds every counter while the SAI handle is still valid -- no
# replay is needed on the new boot. The marker file signals
# finalize-warmboot.sh to re-enable the counter after reconcile.
function disable_flow_cnt_trap_for_warmboot()
{
local key="FLEX_COUNTER_TABLE|FLOW_CNT_TRAP"
local cur
cur=$(sonic-db-cli -n "$NETNS" CONFIG_DB HGET "$key" FLEX_COUNTER_STATUS 2>/dev/null)
if [[ "$cur" != "enable" ]]; then
debug "FLOW_CNT_TRAP not enabled (status='${cur}'), skipping pre-warmboot disable"
# Drop any marker left by a prior aborted warm-reboot so it doesn't
# falsely re-enable a counter the user has since disabled.
rm -f "$FLOW_CNT_TRAP_MARKER"
return
fi
debug "FLOW_CNT_TRAP=enable: disabling pre-warmboot"
mkdir -p "${WARM_DIR}"
touch "$FLOW_CNT_TRAP_MARKER"
sonic-db-cli -n "$NETNS" CONFIG_DB HSET "$key" FLEX_COUNTER_STATUS disable > /dev/null
# Poll COUNTERS_TRAP_NAME_MAP until CoppOrch drains it — that proves every
# SAI_HOSTIF_TRAP_ATTR_COUNTER_ID unbind has completed in orchagent's thread.
local deadline=$((SECONDS + 10))
local n
while (( SECONDS < deadline )); do
n=$(sonic-db-cli -n "$NETNS" COUNTERS_DB HLEN COUNTERS_TRAP_NAME_MAP 2>/dev/null || echo 0)
[[ "$n" == "0" ]] && break
done
if [[ "${n:-0}" != "0" ]]; then
debug "WARN: COUNTERS_TRAP_NAME_MAP still has ${n} entries after 10s; proceeding"
fi
}

# Re-enable FLOW_CNT_TRAP in CONFIG_DB. Used by clear_boot to roll back the
# disable_flow_cnt_trap_for_warmboot CONFIG_DB write when warm-reboot aborts
# before kexec.
function enable_flow_cnt_trap()
{
sonic-db-cli -n "$NETNS" CONFIG_DB HSET "FLEX_COUNTER_TABLE|FLOW_CNT_TRAP" FLEX_COUNTER_STATUS enable > /dev/null
}

# After orchagent pause, there's no rollback if anything goes wrong on Multi-ASIC devices, force execution
if [[ $NUM_ASIC -gt 1 ]]; then
FORCE=yes
fi

if [[ "$REBOOT_TYPE" = "warm-reboot" ]]; then
execute_in_namespaces asic disable_flow_cnt_trap_for_warmboot
fi

if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "express-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
execute_in_namespaces asic pause_orchagent
fi
Expand Down
Loading