diff --git a/config/main.py b/config/main.py index defc401dd8a..93dd6e2ffb2 100644 --- a/config/main.py +++ b/config/main.py @@ -3943,7 +3943,7 @@ def warm_restart_enable(ctx, namespace, module): if namespace is not None: if namespace not in ctx.obj["all_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["all_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"] config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE] feature_table = config_db.get_table('FEATURE') @@ -3966,7 +3966,7 @@ def warm_restart_disable(ctx, namespace, module): if namespace is not None: if namespace not in ctx.obj["all_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["all_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["all_namespaces"] config_db = ctx.obj["config_db"][multi_asic_util.constants.DEFAULT_NAMESPACE] feature_table = config_db.get_table('FEATURE') @@ -3989,7 +3989,7 @@ def warm_restart_neighsyncd_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 9999): @@ -4011,7 +4011,7 @@ def warm_restart_bgp_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 3600): @@ -4033,7 +4033,7 @@ def warm_restart_teamsyncd_timer(ctx, namespace, seconds): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] if ADHOC_VALIDATION: if seconds not in range(1, 3600): @@ -4055,7 +4055,7 @@ def warm_restart_bgp_eoiu(ctx, namespace, enable): if namespace is not None: if namespace not in ctx.obj["asic_namespaces"]: raise click.UsageError("Invalid namespace: {}".format(namespace)) - namespaces = [namespace] if namespace else ctx.obj["asic_namespaces"] + namespaces = [namespace] if namespace is not None else ctx.obj["asic_namespaces"] for namespace in namespaces: db = ValidatedConfigDBConnector(ctx.obj["config_db"][namespace]) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 6d994693a0a..d7fcae466a6 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -131,7 +131,7 @@ function execute_in_namespaces() wait "$pid" && rc=0 || rc=$? # If the command failed, remove the ASIC from the list if [[ $rc -ne 0 ]]; then - error "Command $cmd failed for $dev returned $rc" + error "Command $cmd failed for asic$dev returned $rc" if [[ $FORCE == "no" ]]; then exit $rc else @@ -139,7 +139,7 @@ function execute_in_namespaces() # Remove failed ASIC from the list ASIC_LIST=(${ASIC_LIST[@]/$dev}) # Disable fast/warm-reboot for the failed ASIC - sonic-db-cli -n "asic$dev" HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true + sonic-db-cli -n "asic$dev" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true config warm_restart disable system -n "asic$dev" &>/dev/null || /bin/true fi fi @@ -318,7 +318,11 @@ function filter_asic_list() for asic in "${ASIC_LIST[@]}"; do local skip=0 for skip_asic in "${SKIP_ASICS[@]}"; do - if [[ $asic -eq $skip_asic ]]; then + if [[ ! " ${ASIC_LIST[@]} " =~ " ${skip_asic} " ]]; then + error "Invalid ASIC number to skip: ${skip_asic}, valid ASIC numbers are: ${ASIC_LIST[@]}" + exit "${EXIT_FAILURE}" + fi + if [[ "$asic" == "$skip_asic" ]]; then skip=1 break fi @@ -336,6 +340,12 @@ function filter_asic_list() fi } +function clear_fast_boot_table() +{ + #update FAST_RESTART_ENABLE_TABLE to false + sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true +} + function clear_boot() { # common_clear @@ -361,10 +371,7 @@ function clear_boot() fi done - #clear_fast_boot - if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "false" &>/dev/null || /bin/true - fi + execute_in_namespaces all clear_fast_boot_table } function init_warm_reboot_states() @@ -935,16 +942,20 @@ fi check_conflict_boot_in_fw_update +function enable_fast_boot() +{ + sonic-db-cli -n "$NETNS" STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null || /bin/true + config warm_restart enable -n "$NETNS" system +} # Check reboot type supported BOOT_TYPE_ARG="cold" case "$REBOOT_TYPE" in "fast-reboot") - check_warm_restart_in_progress + execute_in_namespaces all check_warm_restart_in_progress BOOT_TYPE_ARG=$REBOOT_TYPE trap clear_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM - sonic-db-cli STATE_DB HSET "FAST_RESTART_ENABLE_TABLE|system" "enable" "true" &>/dev/null - config warm_restart enable system + execute_in_namespaces all enable_fast_boot ;; "warm-reboot") execute_in_namespaces all check_warm_restart_in_progress @@ -1107,7 +1118,7 @@ function pause_orchagent() docker exec -i swss$DEV /usr/bin/orchagent_restart_check -w 2000 -r 5 > /dev/null || RESTARTCHECK_RC=$? if [[ RESTARTCHECK_RC -ne 0 ]]; then error "RESTARTCHECK failed" - if [[ x"${FORCE}" == x"yes" ]]; then + if [[ x"${FORCE}" == x"yes" && $NUM_ASIC -eq 1 ]]; then debug "Ignoring orchagent pausing failure ..." else exit "${EXIT_ORCHAGENT_SHUTDOWN}" @@ -1134,16 +1145,23 @@ fi # service will go down and we cannot recover from it. set +e -if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - # Clear all routes except of default and connected routes for faster reconciliation time. + +function filter_routes() +{ debug "Clearing routes..." FILTER_ROUTES=0 - python /usr/local/bin/fast-reboot-filter-routes.py || FILTER_ROUTES=$? - if [[ FILTER_ROUTES -ne 0 ]]; then + NAMESPACE_ARG="" + python /usr/local/bin/fast-reboot-filter-routes.py -n "$NETNS" || FILTER_ROUTES=$? + if [[ $FILTER_ROUTES -ne 0 ]]; then error "Preserving connected and default routes failed." else debug "Routes deleted from APP-DB, default and connected routes preserved." fi +} + +if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then + # Clear all routes except of default and connected routes for faster reconciliation time. + execute_in_namespaces asic filter_routes fi # disable trap-handlers which were set before diff --git a/scripts/fast-reboot-filter-routes.py b/scripts/fast-reboot-filter-routes.py index 1e0bcbeca22..5925784e7ee 100755 --- a/scripts/fast-reboot-filter-routes.py +++ b/scripts/fast-reboot-filter-routes.py @@ -1,17 +1,24 @@ #!/usr/bin/env python3 +import argparse import json import sys import os import utilities_common.cli as clicommon import syslog import traceback -from swsscommon.swsscommon import ConfigDBConnector +from swsscommon.swsscommon import ConfigDBConnector, SonicDBConfig +from sonic_py_common import multi_asic ROUTE_IDX = 1 -def get_connected_routes(): - cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"] + +def get_connected_routes(namespace): + if namespace: + ns_name = multi_asic.get_asic_id_from_name(namespace) + cmd = ['sudo', 'vtysh', '-n', ns_name, '-c', "show ip route connected json"] + else: + cmd = ['sudo', 'vtysh', '-c', "show ip route connected json"] connected_routes = [] output, ret = clicommon.run_command(cmd, return_cmd=True) if ret != 0: @@ -23,6 +30,7 @@ def get_connected_routes(): return connected_routes + def get_route(db, route): key = 'ROUTE_TABLE:%s' % route val = db.keys(db.APPL_DB, key) @@ -31,8 +39,9 @@ def get_route(db, route): else: return None -def generate_default_route_entries(): - db = ConfigDBConnector() + +def generate_default_route_entries(namespace): + db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) default_routes = [] @@ -47,8 +56,9 @@ def generate_default_route_entries(): return default_routes -def filter_routes(preserved_routes): - db = ConfigDBConnector() + +def filter_routes(namespace, preserved_routes): + db = ConfigDBConnector(namespace=namespace) db.db_connect(db.APPL_DB) key = 'ROUTE_TABLE:*' @@ -59,13 +69,25 @@ def filter_routes(preserved_routes): if stripped_route not in preserved_routes: db.delete(db.APPL_DB, route) + def main(): - default_routes = generate_default_route_entries() - connected_routes = get_connected_routes() + parser = argparse.ArgumentParser(description='Filter routes for fast-reboot') + parser.add_argument('-n', '--namespace', default=multi_asic.DEFAULT_NAMESPACE, + type=str, help='namespace to use') + args = parser.parse_args() + + namespace = args.namespace if args.namespace else None + + if multi_asic.is_multi_asic(): + SonicDBConfig.initializeGlobalConfig() + + default_routes = generate_default_route_entries(namespace) + connected_routes = get_connected_routes(namespace) preserved_routes = set(default_routes + connected_routes) - filter_routes(preserved_routes) + filter_routes(namespace, preserved_routes) return 0 + if __name__ == '__main__': res = 0 try: diff --git a/scripts/generate_dump b/scripts/generate_dump index 09e60a23a85..bc0758ec425 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -2274,15 +2274,23 @@ collect_bmc_files() { # None ############################################################################### save_warmboot_files() { - # Copy the warmboot files + # Copy the warmboot files (single-ASIC: /host/warmboot; multi-ASIC: /host/warmboot0, /host/warmboot1, ...) trap 'handle_error $? $LINENO' ERR start_t=$(date +%s%3N) if $NOOP; then - echo "$CP $V -rf /host/warmboot $TARDIR" + for d in /host/warmboot*; do + [ -d "$d" ] && echo "$CP $V -rf $d $TARDIR" + done else mkdir -p $TARDIR - $CP $V -rf /host/warmboot $TARDIR - chmod ugo+rw -R $DUMPDIR/$BASE/warmboot + for d in /host/warmboot*; do + if [ -d "$d" ]; then + $CP $V -rf "$d" $TARDIR + fi + done + for d in $TARDIR/warmboot*; do + [ -d "$d" ] && chmod ugo+rw -R "$d" + done fi end_t=$(date +%s%3N) echo "[ Warm-boot Files ] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO diff --git a/tests/fast_reboot_filter_routes_test.py b/tests/fast_reboot_filter_routes_test.py index deb54c2223e..5dc99542032 100644 --- a/tests/fast_reboot_filter_routes_test.py +++ b/tests/fast_reboot_filter_routes_test.py @@ -4,6 +4,7 @@ from mock import patch fast_reboot_filter_routes = importlib.import_module("scripts.fast-reboot-filter-routes") + class TestFastRebootFilterRoutes(object): def setup_method(self): print("SETUP") @@ -11,7 +12,7 @@ def setup_method(self): @patch('utilities_common.cli.run_command') def test_get_connected_routes(self, mock_run_command): mock_run_command.return_value = ('{"1.1.0.0/16": {}}', 0) - output = fast_reboot_filter_routes.get_connected_routes() + output = fast_reboot_filter_routes.get_connected_routes(namespace="") mock_run_command.assert_called_with(['sudo', 'vtysh', '-c', "show ip route connected json"], return_cmd=True) assert output == ['1.1.0.0/16'] @@ -19,8 +20,106 @@ def test_get_connected_routes(self, mock_run_command): def test_get_connected_routes_command_failed(self, mock_run_command): mock_run_command.return_value = ('{"1.1.0.0/16": {}}', 1) with pytest.raises(Exception): - fast_reboot_filter_routes.get_connected_routes() + fast_reboot_filter_routes.get_connected_routes(namespace="") mock_run_command.assert_called_with(['sudo', 'vtysh', '-c', "show ip route connected json"], return_cmd=True) + @patch('utilities_common.cli.run_command') + def test_get_connected_routes_for_namespace(self, mock_run_command): + mock_run_command.return_value = ('{"2.2.0.0/16": {}}', 0) + with mock.patch.object(fast_reboot_filter_routes.multi_asic, + 'get_asic_id_from_name', + return_value='0') as mock_get_asic_id: + output = fast_reboot_filter_routes.get_connected_routes(namespace="asic0") + + mock_get_asic_id.assert_called_once_with("asic0") + mock_run_command.assert_called_with( + ['sudo', 'vtysh', '-n', '0', '-c', "show ip route connected json"], + return_cmd=True + ) + assert output == ['2.2.0.0/16'] + + @patch('utilities_common.cli.run_command') + def test_get_connected_routes_with_empty_output(self, mock_run_command): + mock_run_command.return_value = (None, 0) + output = fast_reboot_filter_routes.get_connected_routes(namespace="") + assert output == [] + + def test_get_route(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = ["ROUTE_TABLE:0.0.0.0/0"] + + output = fast_reboot_filter_routes.get_route(db, "0.0.0.0/0") + + db.keys.assert_called_once_with("APPL_DB", "ROUTE_TABLE:0.0.0.0/0") + assert output == "0.0.0.0/0" + + def test_get_route_not_found(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = [] + + output = fast_reboot_filter_routes.get_route(db, "0.0.0.0/0") + + assert output is None + + def test_generate_default_route_entries(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.side_effect = [ + ["ROUTE_TABLE:0.0.0.0/0"], + ["ROUTE_TABLE:::/0"], + ] + + with mock.patch.object(fast_reboot_filter_routes, + 'ConfigDBConnector', + return_value=db) as mock_config_db: + output = fast_reboot_filter_routes.generate_default_route_entries("asic0") + + mock_config_db.assert_called_once_with(namespace="asic0") + db.db_connect.assert_called_once_with("APPL_DB") + assert output == ["0.0.0.0/0", "::/0"] + + def test_filter_routes(self): + db = mock.Mock() + db.APPL_DB = "APPL_DB" + db.keys.return_value = [ + "ROUTE_TABLE:0.0.0.0/0", + "ROUTE_TABLE:10.0.0.0/24", + "ROUTE_TABLE:192.0.2.0/24", + ] + + with mock.patch.object(fast_reboot_filter_routes, + 'ConfigDBConnector', + return_value=db) as mock_config_db: + fast_reboot_filter_routes.filter_routes("asic0", {"0.0.0.0/0", "10.0.0.0/24"}) + + mock_config_db.assert_called_once_with(namespace="asic0") + db.db_connect.assert_called_once_with("APPL_DB") + db.delete.assert_called_once_with("APPL_DB", "ROUTE_TABLE:192.0.2.0/24") + + def test_main_multi_asic(self): + with mock.patch("sys.argv", ["fast-reboot-filter-routes.py", "-n", "asic0"]), \ + mock.patch.object(fast_reboot_filter_routes.multi_asic, + 'is_multi_asic', + return_value=True), \ + mock.patch.object(fast_reboot_filter_routes.SonicDBConfig, + 'initializeGlobalConfig') as mock_init_config, \ + mock.patch.object(fast_reboot_filter_routes, + 'generate_default_route_entries', + return_value=["0.0.0.0/0"]) as mock_generate_defaults, \ + mock.patch.object(fast_reboot_filter_routes, + 'get_connected_routes', + return_value=["10.0.0.0/24"]) as mock_get_connected, \ + mock.patch.object(fast_reboot_filter_routes, + 'filter_routes') as mock_filter_routes: + output = fast_reboot_filter_routes.main() + + assert output == 0 + mock_init_config.assert_called_once_with() + mock_generate_defaults.assert_called_once_with("asic0") + mock_get_connected.assert_called_once_with("asic0") + mock_filter_routes.assert_called_once_with("asic0", {"0.0.0.0/0", "10.0.0.0/24"}) + def teardown_method(self): print("TEAR DOWN")