Skip to content

[multi-asic][finalize-warmboot] support Multi-ASIC systems #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
stepanblyschak wants to merge 1 commit into from
Closed

[multi-asic][finalize-warmboot] support Multi-ASIC systems #145

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 156 additions & 55 deletions files/image_config/warmboot-finalizer/finalize-warmboot.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
#! /bin/bash

VERBOSE=no
VERBOSE=${VERBOSE:-no}

# read SONiC immutable variables
[ -f /etc/sonic/sonic-environment ] && . /etc/sonic/sonic-environment
PLATFORM=${PLATFORM:-`sonic-cfggen -H -v DEVICE_METADATA.localhost.platform`}

[ -f /usr/share/sonic/device/$PLATFORM/asic.conf ] && . /usr/share/sonic/device/$PLATFORM/asic.conf
NUM_ASIC=${NUM_ASIC:-1}

# Define components that needs to reconcile during warm
# boot:
Expand All @@ -29,16 +33,64 @@ ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"

function debug()
{
/usr/bin/logger "WARMBOOT_FINALIZER : $1"
local message="$1"
if [[ -n $DEV ]]; then
message="asic$DEV: $message"
fi
if [[ x"${VERBOSE}" == x"yes" ]]; then
echo `date` "- $1"
echo `date` "- $message"
fi
/usr/bin/logger "WARMBOOT_FINALIZER : $message"
}

# Determines if a given service has per-ASIC scope.
function is_asic_service()
{
local -r service=$1
local -r has_per_asic_scope=$(sonic-db-cli CONFIG_DB hget "FEATURE|$service" has_per_asic_scope)
[[ x"${has_per_asic_scope,,}" == x"true" ]] && echo 1 || echo 0
}

# Determines if a given service has global scope.
function is_global_service()
{
local -r service=$1
local -r has_global_scope=$(sonic-db-cli CONFIG_DB hget "FEATURE|$service" has_global_scope)
[[ x"${has_global_scope,,}" == x"true" ]] && echo 1 || echo 0
}

# Outputs a list of services that have per-ASIC scope.
function get_asic_service_list()
{
local asic_service_list=""
for service in ${!RECONCILE_COMPONENTS[@]}; do
is_asic_service=$(is_asic_service $service)
if (( is_asic_service )); then
asic_service_list="${asic_service_list} ${service}"
fi
done
echo ${asic_service_list}
}

# Outputs a list of services that have global scope.
function get_global_service_list()
{
local global_service_list=""
for service in ${!RECONCILE_COMPONENTS[@]}; do
is_global_service=$(is_global_service $service)
if (( is_global_service )); then
global_service_list="${global_service_list} ${service}"
fi
done
echo ${global_service_list}
}

ASIC_SERVICE_LIST="$(get_asic_service_list)"
GLOBAL_SERVICE_LIST="$(get_global_service_list)"

function get_component_list()
{
SVC_LIST=${!RECONCILE_COMPONENTS[@]}
SVC_LIST="$1"
COMPONENT_LIST=""
for service in ${SVC_LIST}; do
components=${RECONCILE_COMPONENTS[${service}]}
Expand All @@ -49,16 +101,15 @@ function get_component_list()
done
}


function check_warm_boot()
{
WARM_BOOT=`sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
WARM_BOOT=`sonic-db-cli -n "$NETNS" STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
}

function check_fast_reboot()
{
debug "Checking if fast-reboot is enabled..."
FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
FAST_REBOOT=`sonic-db-cli -n "$NETNS" STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
if [[ x"${FAST_REBOOT}" == x"true" ]]; then
debug "Fast-reboot is enabled..."
else
Expand All @@ -72,12 +123,12 @@ function wait_for_database_service()
debug "Wait for database to become ready..."

# Wait for redis server start before database clean
until [[ $(sonic-db-cli PING | grep -c PONG) -gt 0 ]]; do
until [[ $(sonic-db-cli -n "$NETNS" PING | grep -c PONG) -gt 0 ]]; do
sleep 1;
done

# Wait for configDB initialization
until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]];
until [[ $(sonic-db-cli -n "$NETNS" CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]];
do sleep 1;
done

Expand All @@ -87,7 +138,7 @@ function wait_for_database_service()

function get_component_state()
{
sonic-db-cli STATE_DB hget "WARM_RESTART_TABLE|$1" state
sonic-db-cli -n "$NETNS" STATE_DB hget "WARM_RESTART_TABLE|$1" state
}


Expand All @@ -111,29 +162,44 @@ function set_cpufreq_governor() {
|| debug "Failed to set CPUFreq scaling governor to $governor"
}

function finalize_common() {
local -r asic_type=${ASIC_TYPE:-`sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type`}

if [[ "$asic_type" == "mellanox" ]]; then
# Read default governor from kernel config
local -r default_governor=$(cat "/boot/config-$(uname -r)" | grep -E 'CONFIG_CPU_FREQ_DEFAULT_GOV_.*=y' | sed -E 's/CONFIG_CPU_FREQ_DEFAULT_GOV_(.*)=y/\1/')
set_cpufreq_governor "$default_governor"
fi
}

function finalize_warm_boot()
{
debug "Finalizing warmboot..."
finalize_common
sudo config warm_restart disable
sudo config warm_restart disable -n "$NETNS"
}

function finalize_fast_reboot()
{
debug "Finalizing fast-reboot..."
finalize_common
sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null
sonic-db-cli CONFIG_DB DEL "WARM_RESTART|teamd" &>/dev/null
sonic-db-cli -n "$NETNS" STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null
sonic-db-cli -n "$NETNS" CONFIG_DB DEL "WARM_RESTART|teamd" &>/dev/null
}

# Finalize warm/fast boot in an ASIC namespace.
# This function is called for each ASIC namespace on a Multi-ASIC device.
# For Single-ASIC devices, this function is called as well in global namespace context.
function finalize_boot()
{
if [ x"${FAST_REBOOT}" == x"true" ]; then
finalize_fast_reboot
fi

if [ x"${WARM_BOOT}" == x"true" ]; then
finalize_warm_boot
fi
}

# Finalize warm/fast boot in global namespace.
function finalize_global() {
local -r asic_type=${ASIC_TYPE:-`sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type`}

finalize_boot

if [[ "$asic_type" == "mellanox" ]]; then
# Read default governor from kernel config
local -r default_governor=$(cat "/boot/config-$(uname -r)" | grep -E 'CONFIG_CPU_FREQ_DEFAULT_GOV_.*=y' | sed -E 's/CONFIG_CPU_FREQ_DEFAULT_GOV_(.*)=y/\1/')
set_cpufreq_governor "$default_governor"
fi
}

function stop_control_plane_assistant()
Expand All @@ -155,53 +221,88 @@ function restore_counters_folder()
fi
}

function check_warm_boot_and_fast_boot_or_exit()
{
check_fast_reboot
check_warm_boot
if [[ x"${WARM_BOOT}" != x"true" ]]; then
debug "warmboot is not enabled ..."
if [[ x"${FAST_REBOOT}" != x"true" ]]; then
debug "fastboot is not enabled ..."
exit 0
fi
fi
}

# Wait until a given list of components reach the reconciled state or timeout after 5 minutes.
function wait_for_components_to_reconcile() {
local -r service_list="$1"
get_component_list "${service_list}"

wait_for_database_service
debug "Waiting for components: '${COMPONENT_LIST}' to reconcile ..."

check_fast_reboot
check_warm_boot
list=${COMPONENT_LIST}

if [[ x"${WARM_BOOT}" != x"true" ]]; then
debug "warmboot is not enabled ..."
if [[ x"${FAST_REBOOT}" != x"true" ]]; then
debug "fastboot is not enabled ..."
exit 0
# Wait up to 5 minutes
for i in `seq 60`; do
list=`check_list ${list}`
debug "Waiting for components to reconcile: ${list}"
if [[ -z "${list}" ]]; then
break
fi
sleep 5
done

if [[ -n "${list}" ]]; then
debug "Some components didn't finish reconcile: ${list} ..."
fi
fi
}

wait_for_database_service
check_warm_boot_and_fast_boot_or_exit

if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
restore_counters_folder
fi

get_component_list
# Wait for asic services to reconcile.
# Spawns a separate process for each ASIC to run finalization in parallel.
for dev in `seq 0 $((NUM_ASIC - 1))`; do
(
if [[ $NUM_ASIC -gt 1 ]]; then
export DEV=$dev
export NETNS=asic$dev
fi

debug "Waiting for components: '${COMPONENT_LIST}' to reconcile ..."
# For multi-ASIC devices, wait for the namespace database to be ready
# and check if warm/fast boot is enabled.
if [[ -n $NETNS ]]; then
wait_for_database_service
check_warm_boot_and_fast_boot_or_exit
fi

list=${COMPONENT_LIST}
wait_for_components_to_reconcile "${ASIC_SERVICE_LIST}"

# Wait up to 5 minutes
for i in `seq 60`; do
list=`check_list ${list}`
if [[ -z "${list}" ]]; then
break
fi
sleep 5
# For multi-ASIC devices, finalize the warm/fast boot flags in the namespace database.
if [[ -n $NETNS ]]; then
finalize_boot
fi
) &
done

@oleksandrivantsiv oleksandrivantsiv Dec 19, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we wait for here for processes started in a loop?

@stepanblyschak stepanblyschak Dec 26, 2025 •
edited
Loading

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@oleksandrivantsiv There's wait later:

# Block till all processes complete.
wait

if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
stop_control_plane_assistant
fi
# Wait for global services to reconcile.
# Spawns a separate process to run finalization in parallel.
(
wait_for_components_to_reconcile "${GLOBAL_SERVICE_LIST}"
) &

if [[ -n "${list}" ]]; then
debug "Some components didn't finish reconcile: ${list} ..."
fi
# Block till all processes complete.
wait

if [ x"${FAST_REBOOT}" == x"true" ]; then
finalize_fast_reboot
fi
finalize_global

if [ x"${WARM_BOOT}" == x"true" ]; then
finalize_warm_boot
if [[ (x"${WARM_BOOT}" == x"true") && (x"${FAST_REBOOT}" != x"true") ]]; then
stop_control_plane_assistant
fi

# Save DB after stopped control plane assistant to avoid extra entries
Expand Down