[Multi-ASIC] Added fast-reboot support for multi-asic

files/image_config/config-setup/config-setup

@@ -215,6 +215,23 @@ ztp_is_enabled()
     return $rv
 }
 
+set_config_db_initialized()
+{
+    local value="$1"
+    local asic_num=0
+
+    sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "$value"
+
+    if [[ ${NUM_ASIC:-1} -le 1 ]]; then
+        return 0
+    fi
+
+    while [[ $asic_num -lt $NUM_ASIC ]]; do
+        sonic-db-cli -n "asic$asic_num" CONFIG_DB SET "CONFIG_DB_INITIALIZED" "$value"
+        ((asic_num = asic_num + 1))
+    done
+}
+
 # Generate requested SONiC configuration and save it as destination file
 # Usage: generate_config < factory | ztp > <destination_file>
 #
@@ -341,6 +358,43 @@ get_config_db_file_list()
     echo $config_db_file_list
 }
 
+# Check if a given ASIC namespace has booted warm or fast.
+# On multi-npu platforms individual ASICs can be excluded from a
+# warm/fast reboot, in which case they cold boot while the system as a
+# whole is warm booting.
+check_namespace_warm_boot()
+{
+    local -r namespace="$1"
+    local -r warm_boot=$(sonic-db-cli -n "$namespace" STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable)
+    local -r fast_boot=$(sonic-db-cli -n "$namespace" STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable)
+
+    [[ x"$warm_boot" == x"true" || x"$fast_boot" == x"true" ]]
+}
+
+# Load config db of the ASIC namespaces that were excluded from a
+# warm/fast reboot. Their database was not restored from a redis dump,
+# and the database container could not load the config db file either
+# because it is only restored from the previous image by
+# do_config_migration, which runs after the database container starts.
+load_cold_booted_namespaces()
+{
+    local config_db_file
+    local asic_num=0
+
+    while [[ ($asic_num -lt $NUM_ASIC) && ($NUM_ASIC -gt 1) ]]; do
+        config_db_file=${CONFIG_DB_PATH}${CONFIG_DB_PREFIX}$asic_num${CONFIG_DB_SUFFIX}
+        if ! check_namespace_warm_boot "asic$asic_num" && [ -r $config_db_file ]; then
+            echo "asic$asic_num did not warm reboot, loading ${config_db_file} ..."
+            if [ -r ${INIT_CFG_JSON} ]; then
+                sonic-cfggen -n "asic$asic_num" -j ${INIT_CFG_JSON} -j $config_db_file --write-to-db
+            else
+                sonic-cfggen -n "asic$asic_num" -j $config_db_file --write-to-db
+            fi
+        fi
+        ((asic_num = asic_num + 1))
+    done
+}
+
 # Check if all needed config db are present for both
 # single and multi-npu platforms
 check_all_config_db_present()
@@ -367,7 +421,7 @@ do_db_migration()
         # Migrate the DB to the latest schema version if needed
         /usr/local/bin/db_migrator.py -o migrate
     fi
-    sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
+    set_config_db_initialized "1"
 
     #Enforce CHASSIS_APP_DB.tsa_enabled to be in sync with BGP_DEVICE_GLOBAL.STATE.tsa_enabled
     if [[ -f /etc/sonic/chassisdb.conf ]] && [[ $disaggregated_chassis -ne 1 ]]; then
@@ -404,6 +458,7 @@ do_config_migration()
 
     if [ x"${WARM_BOOT}" == x"true" ]; then
         echo "Warm reboot detected..."
+        load_cold_booted_namespaces
         do_db_migration
         rm -f /etc/sonic/pending_config_migration
         exit 0
@@ -454,7 +509,7 @@ boot_config()
         echo "Warm boot detected, skipping config initialization and ZTP."
         # Mark config as initialized so subsequent boots don't re-trigger
         # initialization unnecessarily (warm-reboot to new image scenario).
-        sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
+        set_config_db_initialized "1"
         rm -f /etc/sonic/pending_config_initialization
         return 0
     fi

files/scripts/bgp.sh

@@ -30,7 +30,7 @@ function validate_restore_count()
 
 function check_fast_boot ()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else

files/scripts/bmp.sh

@@ -18,7 +18,7 @@ function check_warm_boot()
 
 function check_fast_boot ()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else

files/scripts/service_mgmt.sh

@@ -18,7 +18,7 @@ function check_warm_boot()
 
 function check_fast_boot ()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else

files/scripts/swss.sh

@@ -62,7 +62,7 @@ function check_warm_boot()
 
 function check_fast_boot()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
@@ -539,8 +539,8 @@ stop() {
         clean_up_tables STATE_DB "'FABRIC_PORT_TABLE*'"
         debug "Cleared FABRIC_PORT_TABLE from STATE_DB for ${SERVICE}$DEV..."
     else
-        debug "Killing Docker swss..."
-        /usr/bin/docker kill swss &> /dev/null || debug "Docker swss is not running ($?) ..."
+        debug "Killing Docker ${SERVICE}$DEV..."
+        /usr/bin/docker kill ${SERVICE}$DEV &> /dev/null || debug "Docker ${SERVICE}$DEV is not running ($?) ..."
     fi
 
     # Flush FAST_REBOOT table when swss needs to stop. The only
@@ -549,7 +549,7 @@ stop() {
     # be restarted.
     if [[ x"$FAST_BOOT" != x"true" ]]; then
         debug "Clearing FAST_RESTART_ENABLE_TABLE flag..."
-        sonic-db-cli STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false"
+        $SONIC_DB_CLI STATE_DB hset "FAST_RESTART_ENABLE_TABLE|system" "enable" "false"
     fi
     # Unlock has to happen before reaching out to peer service
     unlock_service_state_change

files/scripts/syncd_common.sh

@@ -52,7 +52,7 @@ function check_warm_boot()
 
 function check_fast_boot()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else
@@ -88,7 +88,7 @@ function getBootType()
         ;;
     *SONIC_BOOT_TYPE=fast*|*fast-reboot*)
         # check that the key exists
-        SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+        SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
         if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
             TYPE='fast'
         else

files/scripts/teamd.sh

@@ -32,7 +32,7 @@ function validate_restore_count()
 
 function check_fast_boot ()
 {
-    SYSTEM_FAST_REBOOT=`sonic-db-cli STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
+    SYSTEM_FAST_REBOOT=`$SONIC_DB_CLI STATE_DB hget "FAST_RESTART_ENABLE_TABLE|system" enable`
     if [[ x"${SYSTEM_FAST_REBOOT}" == x"true" ]]; then
         FAST_BOOT="true"
     else

src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/lib.rs

@@ -90,7 +90,7 @@ pub async fn wait_for_container(
         if dependent_services.contains(&container_name) {
             let (service_name, namespace) = parse_container_name(&container_name);
             let warm_restart = device_info::is_warm_restart_enabled_in_namespace(service_name, &namespace)?;
-            let fast_reboot = device_info::is_fast_reboot_enabled()?;
+            let fast_reboot = device_info::is_fast_reboot_enabled_in_namespace(&namespace)?;
 
             if warm_restart || fast_reboot {
                 continue;

src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/tests/docker_wait_any_test.rs

@@ -157,7 +157,7 @@ async fn test_teamd_exits_warm_restart_main_will_not_exit() {
             returns: Ok(true)
         ));
     injector
-        .when_called(func!(sonic_rs_common::device_info::is_fast_reboot_enabled, fn() -> Result<bool, sonic_rs_common::device_info::DeviceInfoError>))
+        .when_called(func!(sonic_rs_common::device_info::is_fast_reboot_enabled_in_namespace, fn(&str) -> Result<bool, sonic_rs_common::device_info::DeviceInfoError>))
         .will_execute(fake!(
             func_type: fn() -> Result<bool, sonic_rs_common::device_info::DeviceInfoError>,
             returns: Ok(false)

src/sonic-rs-common/src/device_info.rs

@@ -84,4 +84,10 @@ pub fn is_fast_reboot_enabled() -> Result<bool> {
     let state_db = StateDBConnector::new()
         .map_err(|e| DeviceInfoError::SwSS(e))?;
     is_fast_reboot_enabled_with_db(&state_db)
+}
+
+pub fn is_fast_reboot_enabled_in_namespace(namespace: &str) -> Result<bool> {
+    let state_db = StateDBConnector::new_with_namespace(namespace)
+        .map_err(|e| DeviceInfoError::SwSS(e))?;
+    is_fast_reboot_enabled_with_db(&state_db)
 }