[Multi-ASIC] Added fast-reboot support for multi-asic

config/main.py

@@ -3943,7 +3943,7 @@ def warm_restart_enable(ctx, namespace, module):
     if namespace is not None:
         if namespace not in ctx.obj["all_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["all_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"]
 
     config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE]
     feature_table = config_db.get_table('FEATURE')
@@ -3966,7 +3966,7 @@ def warm_restart_disable(ctx, namespace, module):
     if namespace is not None:
         if namespace not in ctx.obj["all_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["all_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"]
 
     config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE]
     feature_table = config_db.get_table('FEATURE')
@@ -3989,7 +3989,7 @@ def warm_restart_neighsyncd_timer(ctx, namespace, seconds):
     if namespace is not None:
         if namespace not in ctx.obj["asic_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"]
 
     if ADHOC_VALIDATION:
         if seconds not in range(1, 9999):
@@ -4011,7 +4011,7 @@ def warm_restart_bgp_timer(ctx, namespace, seconds):
     if namespace is not None:
         if namespace not in ctx.obj["asic_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"]
 
     if ADHOC_VALIDATION:
         if seconds not in range(1, 3600):
@@ -4033,7 +4033,7 @@ def warm_restart_teamsyncd_timer(ctx, namespace, seconds):
     if namespace is not None:
         if namespace not in ctx.obj["asic_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"]
 
     if ADHOC_VALIDATION:
         if seconds not in range(1, 3600):
@@ -4055,7 +4055,7 @@ def warm_restart_bgp_eoiu(ctx, namespace, enable):
     if namespace is not None:
         if namespace not in ctx.obj["asic_namespaces"]:
             raise click.UsageError("Invalid namespace: {}".format(namespace))
-    namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"]
+    namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"]
 
     for namespace in namespaces:
         db = ValidatedConfigDBConnector(ctx.obj["config_db"][namespace])

scripts/fast-reboot

@@ -131,15 +131,15 @@ function execute_in_namespaces()
             wait "$pid" && rc=0 || rc=$?
             # If the command failed, remove the ASIC from the list
             if [[ $rc -ne 0 ]]; then
-                error "Command $cmd failed for $dev returned $rc"
+                error "Command $cmd failed for asic$dev returned $rc"
                 if [[ $FORCE == "no" ]]; then
                     exit $rc
                 else
                     debug "Force disabling fast/warm-reboot for asic$dev"
                     # Remove failed ASIC from the list
                     ASIC_LIST=(${ASIC_LIST[@]/$dev})
                     # Disable fast/warm-reboot for the failed ASIC
-                    sonic-db-cli -n "asic$dev" HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true
+                    sonic-db-cli -n "asic$dev" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true
                     config warm_restart disable system -n "asic$dev" &>/dev/null || /bin/true
                 fi
             fi
@@ -318,7 +318,11 @@ function filter_asic_list()
         for asic in "${ASIC_LIST[@]}"; do
             local skip=0
             for skip_asic in "${SKIP_ASICS[@]}"; do
-                if [[ $asic -eq $skip_asic ]]; then
+                if [[ ! " ${ASIC_LIST[@]} " =~ " ${skip_asic} " ]]; then
+                    error "Invalid ASIC number to skip: ${skip_asic}, valid ASIC numbers are: ${ASIC_LIST[@]}"
+                    exit "${EXIT_FAILURE}"
+                fi
+                if [[ "$asic" == "$skip_asic" ]]; then
                     skip=1
                     break
                 fi
@@ -336,6 +340,12 @@ function filter_asic_list()
     fi
 }
 
+function clear_fast_boot_table()
+{
+    #update FAST_RESTART_ENABLE_TABLE to false
+    sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true
+}
+
 function clear_boot()
 {
     # common_clear
@@ -361,10 +371,7 @@ function clear_boot()
         fi
     done
 
-    #clear_fast_boot
-    if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
-        sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true
-    fi
+    execute_in_namespaces all clear_fast_boot_table
 }
 
 function init_warm_reboot_states()
@@ -935,16 +942,20 @@ fi
 
 check_conflict_boot_in_fw_update
 
+function enable_fast_boot()
+{
+    sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null || /bin/true
+    config warm_restart enable -n "$NETNS" system
+}
 
 # Check reboot type supported
 BOOT_TYPE_ARG="cold"
 case "$REBOOT_TYPE" in
     "fast-reboot")
-        check_warm_restart_in_progress
+        execute_in_namespaces all check_warm_restart_in_progress
         BOOT_TYPE_ARG=$REBOOT_TYPE
         trap clear_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
-        sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null
-        config warm_restart enable system
+        execute_in_namespaces all enable_fast_boot
         ;;
     "warm-reboot")
         execute_in_namespaces all check_warm_restart_in_progress
@@ -1107,7 +1118,7 @@ function pause_orchagent()
     docker exec -i swss$DEV /usr/bin/orchagent_restart_check -w 2000 -r 5 > /dev/null || RESTARTCHECK_RC=$?
     if [[ RESTARTCHECK_RC -ne 0 ]]; then
         error "RESTARTCHECK failed"
-        if [[ x"${FORCE}" == x"yes" ]]; then
+        if [[ x"${FORCE}" == x"yes" && $NUM_ASIC -eq 1 ]]; then
             debug "Ignoring orchagent pausing failure ..."
         else
             exit "${EXIT_ORCHAGENT_SHUTDOWN}"
@@ -1134,16 +1145,23 @@ fi
 # service will go down and we cannot recover from it.
 set +e
 
-if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
-    # Clear all routes except of default and connected routes for faster reconciliation time.
+
+function filter_routes()
+{
     debug "Clearing routes..."
     FILTER_ROUTES=0
-    python /usr/local/bin/fast-reboot-filter-routes.py || FILTER_ROUTES=$?
-    if [[ FILTER_ROUTES -ne 0 ]]; then
+    NAMESPACE_ARG=""
+    python /usr/local/bin/fast-reboot-filter-routes.py -n "$NETNS" || FILTER_ROUTES=$?
+    if [[ $FILTER_ROUTES -ne 0 ]]; then
         error "Preserving connected and default routes failed."
     else
         debug "Routes deleted from APP-DB, default and connected routes preserved."
     fi
+}
+
+if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
+    # Clear all routes except of default and connected routes for faster reconciliation time.
+    execute_in_namespaces asic filter_routes
 fi
 
 # disable trap-handlers which were set before

scripts/fast-reboot-filter-routes.py

@@ -1,17 +1,24 @@
 #!/usr/bin/env python3
 
+import argparse
 import json
 import sys
 import os
 import utilities_common.cli as clicommon
 import syslog
 import traceback
-from swsscommon.swsscommon import ConfigDBConnector
+from swsscommon.swsscommon import ConfigDBConnector, SonicDBConfig
+from sonic_py_common import multi_asic
 
 ROUTE_IDX = 1
 
-def get_connected_routes():
-    cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"]
+
+def get_connected_routes(namespace):
+    if namespace:
+        ns_name = multi_asic.get_asic_id_from_name(namespace)
+        cmd = ['sudo', 'vtysh', '-n', ns_name, '-c', "show ip route connected json"]
+    else:
+        cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"]
     connected_routes = []
     output, ret = clicommon.run_command(cmd, return_cmd=True)
     if ret != 0:
@@ -23,6 +30,7 @@ def get_connected_routes():
 
     return connected_routes
 
+
 def get_route(db, route):
     key = 'ROUTE_TABLE:%s' % route
     val = db.keys(db.APPL_DB, key)
@@ -31,8 +39,9 @@ def get_route(db, route):
     else:
         return None
 
-def generate_default_route_entries():
-    db = ConfigDBConnector()
+
+def generate_default_route_entries(namespace):
+    db = ConfigDBConnector(namespace=namespace)
     db.db_connect(db.APPL_DB)
 
     default_routes = []
@@ -47,8 +56,9 @@ def generate_default_route_entries():
 
     return default_routes
 
-def filter_routes(preserved_routes):
-    db = ConfigDBConnector()
+
+def filter_routes(namespace, preserved_routes):
+    db = ConfigDBConnector(namespace=namespace)
     db.db_connect(db.APPL_DB)
 
     key = 'ROUTE_TABLE:*'
@@ -59,13 +69,25 @@ def filter_routes(preserved_routes):
         if stripped_route not in preserved_routes:
             db.delete(db.APPL_DB, route)
 
+
 def main():
-    default_routes = generate_default_route_entries()
-    connected_routes = get_connected_routes()
+    parser = argparse.ArgumentParser(description='Filter routes for fast-reboot')
+    parser.add_argument('-n', '--namespace', default=multi_asic.DEFAULT_NAMESPACE,
+                        type=str, help='namespace to use')
+    args = parser.parse_args()
+
+    namespace = args.namespace if args.namespace else None
+
+    if multi_asic.is_multi_asic():
+        SonicDBConfig.initializeGlobalConfig()
+
+    default_routes = generate_default_route_entries(namespace)
+    connected_routes = get_connected_routes(namespace)
     preserved_routes = set(default_routes + connected_routes)
-    filter_routes(preserved_routes)
+    filter_routes(namespace, preserved_routes)
     return 0
 
+
 if __name__ == '__main__':
     res = 0
     try:

scripts/generate_dump

@@ -2274,15 +2274,23 @@ collect_bmc_files() {
 #  None
 ###############################################################################
 save_warmboot_files() {
-    # Copy the warmboot files
+    # Copy the warmboot files (single-ASIC: /host/warmboot; multi-ASIC: /host/warmboot0, /host/warmboot1, ...)
     trap 'handle_error $? $LINENO' ERR
     start_t=$(date +%s%3N)
     if $NOOP; then
-        echo "$CP $V -rf /host/warmboot $TARDIR"
+        for d in /host/warmboot*; do
+            [ -d "$d" ] && echo "$CP $V -rf $d $TARDIR"
+        done
     else
         mkdir -p $TARDIR
-        $CP $V -rf /host/warmboot $TARDIR
-        chmod ugo+rw -R $DUMPDIR/$BASE/warmboot
+        for d in /host/warmboot*; do
+            if [ -d "$d" ]; then
+                $CP $V -rf "$d" $TARDIR
+            fi
+        done
+        for d in $TARDIR/warmboot*; do
+            [ -d "$d" ] && chmod ugo+rw -R "$d"
+        done
     fi
     end_t=$(date +%s%3N)
     echo "[ Warm-boot Files ] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO