From 91a71c52c67408b18978f290aea8b6d7da28b807 Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Sun, 25 Jan 2026 11:18:54 +0200 Subject: [PATCH 1/7] Fast reboot | Added multi-ASIC support Signed-off-by: Yair Raviv --- scripts/fast-reboot | 36 +++++++++++++++++-------- scripts/fast-reboot-filter-routes.py | 36 ++++++++++++++++++------- scripts/generate_dump | 16 ++++++++--- tests/fast_reboot_filter_routes_test.py | 4 +-- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 6d994693a0a..32310e5622f 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -336,6 +336,12 @@ function filter_asic_list() fi } +function clear_fast_boot_table() +{ + #update FAST_RESTART_ENABLE_TABLE to false + sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true +} + function clear_boot() { # common_clear @@ -361,10 +367,7 @@ function clear_boot() fi done - #clear_fast_boot - if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true - fi + execute_in_namespaces all clear_fast_boot_table } function init_warm_reboot_states() @@ -935,16 +938,20 @@ fi check_conflict_boot_in_fw_update +function enable_fast_boot() +{ + sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null || /bin/true + config warm_restart enable -n "$NETNS" system +} # Check reboot type supported BOOT_TYPE_ARG="cold" case "$REBOOT_TYPE" in "fast-reboot") - check_warm_restart_in_progress + execute_in_namespaces all check_warm_restart_in_progress BOOT_TYPE_ARG=$REBOOT_TYPE trap clear_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM - sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null - config warm_restart enable system + execute_in_namespaces all enable_fast_boot ;; "warm-reboot") execute_in_namespaces all check_warm_restart_in_progress @@ -1134,16 +1141,23 @@ fi # service will go down and we cannot recover from it. set +e -if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - # Clear all routes except of default and connected routes for faster reconciliation time. + +function filter_routes() +{ debug "Clearing routes..." FILTER_ROUTES=0 - python /usr/local/bin/fast-reboot-filter-routes.py || FILTER_ROUTES=$? - if [[ FILTER_ROUTES -ne 0 ]]; then + NAMESPACE_ARG="" + python /usr/local/bin/fast-reboot-filter-routes.py -n $NETNS || FILTER_ROUTES=$? + if [[ $FILTER_ROUTES -ne 0 ]]; then error "Preserving connected and default routes failed." else debug "Routes deleted from APP-DB, default and connected routes preserved." fi +} + +if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then + # Clear all routes except of default and connected routes for faster reconciliation time. + execute_in_namespaces asic filter_routes fi # disable trap-handlers which were set before diff --git a/scripts/fast-reboot-filter-routes.py b/scripts/fast-reboot-filter-routes.py index 1e0bcbeca22..e7729a15ae8 100755 --- a/scripts/fast-reboot-filter-routes.py +++ b/scripts/fast-reboot-filter-routes.py @@ -1,17 +1,23 @@ #!/usr/bin/env python3 +import argparse import json import sys import os import utilities_common.cli as clicommon import syslog import traceback -from swsscommon.swsscommon import ConfigDBConnector +from swsscommon.swsscommon import ConfigDBConnector, SonicDBConfig +from sonic_py_common import multi_asic ROUTE_IDX = 1 -def get_connected_routes(): - cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"] +def get_connected_routes(namespace): + if namespace: + ns_name = multi_asic.get_asic_id_from_name(namespace) + cmd = ['sudo', 'vtysh', '-n', ns_name, '-c', "show ip route connected json"] + else: + cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"] connected_routes = [] output, ret = clicommon.run_command(cmd, return_cmd=True) if ret != 0: @@ -31,8 +37,8 @@ def get_route(db, route): else: return None -def generate_default_route_entries(): - db = ConfigDBConnector() +def generate_default_route_entries(namespace): + db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) default_routes = [] @@ -47,8 +53,8 @@ def generate_default_route_entries(): return default_routes -def filter_routes(preserved_routes): - db = ConfigDBConnector() +def filter_routes(namespace, preserved_routes): + db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) key = 'ROUTE_TABLE:*' @@ -60,10 +66,20 @@ def filter_routes(preserved_routes): db.delete(db.APPL_DB, route) def main(): - default_routes = generate_default_route_entries() - connected_routes = get_connected_routes() + parser = argparse.ArgumentParser(description='Filter routes for fast-reboot') + parser.add_argument('-n', '--namespace', default=multi_asic.DEFAULT_NAMESPACE, + type=str, help='namespace to use') + args = parser.parse_args() + + namespace = args.namespace if args.namespace else None + + if multi_asic.is_multi_asic(): + SonicDBConfig.initializeGlobalConfig() + + default_routes = generate_default_route_entries(namespace) + connected_routes = get_connected_routes(namespace) preserved_routes = set(default_routes + connected_routes) - filter_routes(preserved_routes) + filter_routes(namespace, preserved_routes) return 0 if __name__ == '__main__': diff --git a/scripts/generate_dump b/scripts/generate_dump index 09e60a23a85..bc0758ec425 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -2274,15 +2274,23 @@ collect_bmc_files() { # None ############################################################################### save_warmboot_files() { - # Copy the warmboot files + # Copy the warmboot files (single-ASIC: /host/warmboot; multi-ASIC: /host/warmboot0, /host/warmboot1, ...) trap 'handle_error $? $LINENO' ERR start_t=$(date +%s%3N) if $NOOP; then - echo "$CP $V -rf /host/warmboot $TARDIR" + for d in /host/warmboot*; do + [ -d "$d" ] && echo "$CP $V -rf $d $TARDIR" + done else mkdir -p $TARDIR - $CP $V -rf /host/warmboot $TARDIR - chmod ugo+rw -R $DUMPDIR/$BASE/warmboot + for d in /host/warmboot*; do + if [ -d "$d" ]; then + $CP $V -rf "$d" $TARDIR + fi + done + for d in $TARDIR/warmboot*; do + [ -d "$d" ] && chmod ugo+rw -R "$d" + done fi end_t=$(date +%s%3N) echo "[ Warm-boot Files ] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO diff --git a/tests/fast_reboot_filter_routes_test.py b/tests/fast_reboot_filter_routes_test.py index deb54c2223e..187da5b5cbb 100644 --- a/tests/fast_reboot_filter_routes_test.py +++ b/tests/fast_reboot_filter_routes_test.py @@ -11,7 +11,7 @@ def setup_method(self): @patch('utilities_common.cli.run_command') def test_get_connected_routes(self, mock_run_command): mock_run_command.return_value = ('{"1.1.0.0/16": {}}', 0) - output = fast_reboot_filter_routes.get_connected_routes() + output = fast_reboot_filter_routes.get_connected_routes(namespace="") mock_run_command.assert_called_with(['sudo', 'vtysh', '-c', "show ip route connected json"], return_cmd=True) assert output == ['1.1.0.0/16'] @@ -19,7 +19,7 @@ def test_get_connected_routes(self, mock_run_command): def test_get_connected_routes_command_failed(self, mock_run_command): mock_run_command.return_value = ('{"1.1.0.0/16": {}}', 1) with pytest.raises(Exception): - fast_reboot_filter_routes.get_connected_routes() + fast_reboot_filter_routes.get_connected_routes(namespace="") mock_run_command.assert_called_with(['sudo', 'vtysh', '-c', "show ip route connected json"], return_cmd=True) def teardown_method(self): From 5db37bc4ce59fec0d2b8677a971aab1f2e680ffb Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Wed, 11 Mar 2026 20:45:00 +0200 Subject: [PATCH 2/7] Fixed cleanup logic for 1 ASIC failure Signed-off-by: Yair Raviv --- scripts/fast-reboot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 32310e5622f..f2e3a0c16ca 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -131,7 +131,7 @@ function execute_in_namespaces() wait "$pid" && rc=0 || rc=$? # If the command failed, remove the ASIC from the list if [[ $rc -ne 0 ]]; then - error "Command $cmd failed for $dev returned $rc" + error "Command $cmd failed for asic$dev returned $rc" if [[ $FORCE == "no" ]]; then exit $rc else @@ -139,7 +139,7 @@ function execute_in_namespaces() # Remove failed ASIC from the list ASIC_LIST=(${ASIC_LIST[@]/$dev}) # Disable fast/warm-reboot for the failed ASIC - sonic-db-cli -n "asic$dev" HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true + sonic-db-cli -n "asic$dev" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true config warm_restart disable system -n "asic$dev" &>/dev/null || /bin/true fi fi @@ -1114,7 +1114,7 @@ function pause_orchagent() docker exec -i swss$DEV /usr/bin/orchagent_restart_check -w 2000 -r 5 > /dev/null || RESTARTCHECK_RC=$? if [[ RESTARTCHECK_RC -ne 0 ]]; then error "RESTARTCHECK failed" - if [[ x"${FORCE}" == x"yes" ]]; then + if [[ x"${FORCE}" == x"yes" && $NUM_ASIC -eq 1 ]]; then debug "Ignoring orchagent pausing failure ..." else exit "${EXIT_ORCHAGENT_SHUTDOWN}" From c0027fe998c06e49758f8fd2ee7ef0d375c77a95 Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Mon, 30 Mar 2026 13:25:38 +0300 Subject: [PATCH 3/7] Fixed filter-routes call for single-asic Signed-off-by: Yair Raviv --- scripts/fast-reboot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index f2e3a0c16ca..a6addc639bd 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -1147,7 +1147,7 @@ function filter_routes() debug "Clearing routes..." FILTER_ROUTES=0 NAMESPACE_ARG="" - python /usr/local/bin/fast-reboot-filter-routes.py -n $NETNS || FILTER_ROUTES=$? + python /usr/local/bin/fast-reboot-filter-routes.py -n "$NETNS" || FILTER_ROUTES=$? if [[ $FILTER_ROUTES -ne 0 ]]; then error "Preserving connected and default routes failed." else From 88125530d21de91d41a8b434b402cfa87de28b05 Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Sun, 5 Apr 2026 14:51:06 +0300 Subject: [PATCH 4/7] Added validation to fast/warm boot -m flag values (ASIC numbers) Signed-off-by: Yair Raviv --- scripts/fast-reboot | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index a6addc639bd..d7fcae466a6 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -318,7 +318,11 @@ function filter_asic_list() for asic in "${ASIC_LIST[@]}"; do local skip=0 for skip_asic in "${SKIP_ASICS[@]}"; do - if [[ $asic -eq $skip_asic ]]; then + if [[ ! " ${ASIC_LIST[@]} " =~ " ${skip_asic} " ]]; then + error "Invalid ASIC number to skip: ${skip_asic}, valid ASIC numbers are: ${ASIC_LIST[@]}" + exit "${EXIT_FAILURE}" + fi + if [[ "$asic" == "$skip_asic" ]]; then skip=1 break fi From 405ea24e58eeb0c677761baa46e889ca7ab69053 Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Sun, 26 Apr 2026 17:17:12 +0300 Subject: [PATCH 5/7] Cosmetic fix Signed-off-by: Yair Raviv --- scripts/fast-reboot-filter-routes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/fast-reboot-filter-routes.py b/scripts/fast-reboot-filter-routes.py index e7729a15ae8..0fac54a9b34 100755 --- a/scripts/fast-reboot-filter-routes.py +++ b/scripts/fast-reboot-filter-routes.py @@ -12,6 +12,7 @@ ROUTE_IDX = 1 + def get_connected_routes(namespace): if namespace: ns_name = multi_asic.get_asic_id_from_name(namespace) @@ -29,6 +30,7 @@ def get_connected_routes(namespace): return connected_routes + def get_route(db, route): key = 'ROUTE_TABLE:%s' % route val = db.keys(db.APPL_DB, key) @@ -37,6 +39,7 @@ def get_route(db, route): else: return None + def generate_default_route_entries(namespace): db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) @@ -65,6 +68,7 @@ def filter_routes(namespace, preserved_routes): if stripped_route not in preserved_routes: db.delete(db.APPL_DB, route) + def main(): parser = argparse.ArgumentParser(description='Filter routes for fast-reboot') parser.add_argument('-n', '--namespace', default=multi_asic.DEFAULT_NAMESPACE, @@ -82,6 +86,7 @@ def main(): filter_routes(namespace, preserved_routes) return 0 + if __name__ == '__main__': res = 0 try: From cf481503c6e1618b6e4e6f86548a0e541dd73c2b Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Mon, 27 Apr 2026 16:43:55 +0300 Subject: [PATCH 6/7] Added coverage Signed-off-by: Yair Raviv --- scripts/fast-reboot-filter-routes.py | 1 + tests/fast_reboot_filter_routes_test.py | 99 +++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/scripts/fast-reboot-filter-routes.py b/scripts/fast-reboot-filter-routes.py index 0fac54a9b34..5925784e7ee 100755 --- a/scripts/fast-reboot-filter-routes.py +++ b/scripts/fast-reboot-filter-routes.py @@ -56,6 +56,7 @@ def generate_default_route_entries(namespace): return default_routes + def filter_routes(namespace, preserved_routes): db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) diff --git a/tests/fast_reboot_filter_routes_test.py b/tests/fast_reboot_filter_routes_test.py index 187da5b5cbb..5dc99542032 100644 --- a/tests/fast_reboot_filter_routes_test.py +++ b/tests/fast_reboot_filter_routes_test.py @@ -4,6 +4,7 @@ from mock import patch fast_reboot_filter_routes = importlib.import_module("scripts.fast-reboot-filter-routes") + class TestFastRebootFilterRoutes(object): def setup_method(self): print("SETUP") @@ -22,5 +23,103 @@ def test_get_connected_routes_command_failed(self, mock_run_command): fast_reboot_filter_routes.get_connected_routes(namespace="") mock_run_command.assert_called_with(['sudo', 'vtysh', '-c', "show ip route connected json"], return_cmd=True) + @patch('utilities_common.cli.run_command') + def test_get_connected_routes_for_namespace(self, mock_run_command): + mock_run_command.return_value = ('{"2.2.0.0/16": {}}', 0) + with mock.patch.object(fast_reboot_filter_routes.multi_asic, + 'get_asic_id_from_name', + return_value='0') as mock_get_asic_id: + output = fast_reboot_filter_routes.get_connected_routes(namespace="asic0") + + mock_get_asic_id.assert_called_once_with("asic0") + mock_run_command.assert_called_with( + ['sudo', 'vtysh', '-n', '0', '-c', "show ip route connected json"], + return_cmd=True + ) + assert output == ['2.2.0.0/16'] + + @patch('utilities_common.cli.run_command') + def test_get_connected_routes_with_empty_output(self, mock_run_command): + mock_run_command.return_value = (None, 0) + output = fast_reboot_filter_routes.get_connected_routes(namespace="") + assert output == [] + + def test_get_route(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = ["ROUTE_TABLE:0.0.0.0/0"] + + output = fast_reboot_filter_routes.get_route(db, "0.0.0.0/0") + + db.keys.assert_called_once_with("APPL_DB", "ROUTE_TABLE:0.0.0.0/0") + assert output == "0.0.0.0/0" + + def test_get_route_not_found(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = [] + + output = fast_reboot_filter_routes.get_route(db, "0.0.0.0/0") + + assert output is None + + def test_generate_default_route_entries(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.side_effect = [ + ["ROUTE_TABLE:0.0.0.0/0"], + ["ROUTE_TABLE:::/0"], + ] + + with mock.patch.object(fast_reboot_filter_routes, + 'ConfigDBConnector', + return_value=db) as mock_config_db: + output = fast_reboot_filter_routes.generate_default_route_entries("asic0") + + mock_config_db.assert_called_once_with(namespace="asic0") + db.db_connect.assert_called_once_with("APPL_DB") + assert output == ["0.0.0.0/0", "::/0"] + + def test_filter_routes(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = [ + "ROUTE_TABLE:0.0.0.0/0", + "ROUTE_TABLE:10.0.0.0/24", + "ROUTE_TABLE:192.0.2.0/24", + ] + + with mock.patch.object(fast_reboot_filter_routes, + 'ConfigDBConnector', + return_value=db) as mock_config_db: + fast_reboot_filter_routes.filter_routes("asic0", {"0.0.0.0/0", "10.0.0.0/24"}) + + mock_config_db.assert_called_once_with(namespace="asic0") + db.db_connect.assert_called_once_with("APPL_DB") + db.delete.assert_called_once_with("APPL_DB", "ROUTE_TABLE:192.0.2.0/24") + + def test_main_multi_asic(self): + with mock.patch("sys.argv", ["fast-reboot-filter-routes.py", "-n", "asic0"]), \ + mock.patch.object(fast_reboot_filter_routes.multi_asic, + 'is_multi_asic', + return_value=True), \ + mock.patch.object(fast_reboot_filter_routes.SonicDBConfig, + 'initializeGlobalConfig') as mock_init_config, \ + mock.patch.object(fast_reboot_filter_routes, + 'generate_default_route_entries', + return_value=["0.0.0.0/0"]) as mock_generate_defaults, \ + mock.patch.object(fast_reboot_filter_routes, + 'get_connected_routes', + return_value=["10.0.0.0/24"]) as mock_get_connected, \ + mock.patch.object(fast_reboot_filter_routes, + 'filter_routes') as mock_filter_routes: + output = fast_reboot_filter_routes.main() + + assert output == 0 + mock_init_config.assert_called_once_with() + mock_generate_defaults.assert_called_once_with("asic0") + mock_get_connected.assert_called_once_with("asic0") + mock_filter_routes.assert_called_once_with("asic0", {"0.0.0.0/0", "10.0.0.0/24"}) + def teardown_method(self): print("TEAR DOWN") From a23f464e7ac7e1d7023712a1a65b224b5f35f235 Mon Sep 17 00:00:00 2001 From: Yair Raviv Date: Tue, 28 Apr 2026 14:03:06 +0300 Subject: [PATCH 7/7] Fix enable warm reboot on global namespaces Signed-off-by: Yair Raviv --- config/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/main.py b/config/main.py index defc401dd8a..93dd6e2ffb2 100644 --- a/config/main.py +++ b/config/main.py @@ -3943,7 +3943,7 @@ def warm_restart_enable(ctx, namespace, module): if namespace is not None: if namespace not in ctx.obj["all_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["all_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"] config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE] feature_table = config_db.get_table('FEATURE') @@ -3966,7 +3966,7 @@ def warm_restart_disable(ctx, namespace, module): if namespace is not None: if namespace not in ctx.obj["all_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["all_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"] config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE] feature_table = config_db.get_table('FEATURE') @@ -3989,7 +3989,7 @@ def warm_restart_neighsyncd_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 9999): @@ -4011,7 +4011,7 @@ def warm_restart_bgp_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 3600): @@ -4033,7 +4033,7 @@ def warm_restart_teamsyncd_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 3600): @@ -4055,7 +4055,7 @@ def warm_restart_bgp_eoiu(ctx, namespace, enable): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] for namespace in namespaces: db = ValidatedConfigDBConnector(ctx.obj["config_db"][namespace])