From 52032f91d6a2caa913427199e4544d3fbeaf8bdb Mon Sep 17 00:00:00 2001 From: Sonic Build Admin Date: Wed, 6 May 2026 06:44:13 +0000 Subject: [PATCH] [bgp/agg]: Add BGP aggregate address test cases for Config Persistence and Recovery ### Description of PR Summary: Add new test file `test_bgp_aggregate_address_resilience.py` (Test Group 5) that validates BGP aggregate-address configuration persistence and recovery across various disruption scenarios. These 5 new test cases verify that aggregate address configuration written via GCU survives BGP container restarts, config reloads, cold reboots, warm reboots, and BBR state transitions. ADO: 37526899 New test cases: - **TC 5.1** `test_aggregate_persists_bgp_container_restart`: Aggregate config survives BGP container restart; CONFIG_DB + STATE_DB + FRR are consistent after recovery. - **TC 5.2** `test_aggregate_persists_config_reload`: Aggregate config (with summary-only=true) survives config save + config reload. - **TC 5.3** `test_aggregate_persists_config_save_and_reboot`: IPv6 aggregate config survives config save + cold reboot. - **TC 5.4** `test_aggregate_bbr_required_inactive_persists_bgp_restart`: BBR-required aggregate stays inactive after BGP restart when BBR is disabled; activates once BBR is enabled. - **TC 5.5** `test_aggregate_persists_warm_reboot`: Aggregate config survives warm reboot. ### Type of change - [ ] Bug fix - [ ] Testbed and Framework(new/improvement) - [x] New Test case - [x] Skipped for non-supported platforms - [ ] Test case improvement ### Back port request - [ ] 202205 - [ ] 202305 - [ ] 202311 - [ ] 202405 - [ ] 202411 - [ ] 202505 - [ ] 202511 ### Approach #### What is the motivation for this PR? Existing BGP aggregate-address tests cover configuration validation and route propagation behavior, but there are no tests verifying that aggregate address configuration persists across operational disruptions such as BGP container restarts, config reloads, and device reboots. This PR fills that gap by adding resilience tests that validate CONFIG_DB, STATE_DB, and FRR running-config consistency after each disruption type. #### How did you do it? - Created `test_bgp_aggregate_address_resilience.py` reusing existing helpers and fixtures from `test_bgp_aggregate_address.py` (`AggregateCfg`, `gcu_add_aggregate`, `gcu_remove_aggregate`, `verify_bgp_aggregate_consistence`, `verify_bgp_aggregate_cleanup`, `dump_db`, and the `setup_teardown` checkpoint/rollback fixture). - Added a `bgp_neighbors` fixture to discover BGP neighbor IPs for session-state polling after disruptions. - Pre-disruption verification only checks CONFIG_DB (GCU write is synchronous). Post-disruption verification checks the full stack (CONFIG_DB + STATE_DB + FRR) after bgpcfgd has re-processed the config. - Added `wait_for_aggregate_state()` helper to handle the asynchronous bgpcfgd STATE_DB population after disruptions. - All test cases include proper cleanup in `finally` blocks with graceful fallback to checkpoint rollback. #### How did you verify/test it? Ran all test cases on a physical m1-48 testbed with Arista EOS neighbors. image #### Any platform specific information? No platform-specific dependencies. Tests use GCU for configuration and standard SONiC reboot/reload utilities, which are platform-agnostic. #### Supported testbed topology if it's a new test case? t1, m1 (declared via `@pytest.mark.topology("t1", "m1")`) ### Documentation Aligned with [BGP-Aggregate-Address test plan](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testplan/BGP-Aggregate-Address.md) Signed-off-by: Sonic Build Admin --- .../test_bgp_aggregate_address_resilience.py | 355 ++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 tests/bgp/test_bgp_aggregate_address_resilience.py diff --git a/tests/bgp/test_bgp_aggregate_address_resilience.py b/tests/bgp/test_bgp_aggregate_address_resilience.py new file mode 100644 index 0000000000..e0f89a4428 --- /dev/null +++ b/tests/bgp/test_bgp_aggregate_address_resilience.py @@ -0,0 +1,355 @@ +""" +Tests for BGP aggregate-address config persistence and recovery (Test Group 5). + +Validates that aggregate address configuration survives disruptions: + TC 5.1: BGP container restart + TC 5.2: Config reload + TC 5.3: Config save and cold reboot + TC 5.4: BGP container restart with BBR-required inactive aggregate + TC 5.5: Warm reboot + +Aligned with: + https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testplan/BGP-Aggregate-Address.md + +Verification approach: + Pre-disruption: verify CONFIG_DB only (GCU write is synchronous). + Post-disruption: full CONFIG_DB + STATE_DB + FRR running-config check + (the disruption restarts bgpcfgd, which populates STATE_DB and FRR). +""" + +import logging + +import pytest + +from bgp_bbr_helpers import config_bbr_by_gcu, get_bbr_default_state, is_bbr_enabled + +from test_bgp_aggregate_address import ( + AggregateCfg, + AGGR_V4, + AGGR_V6, + BGP_AGGREGATE_ADDRESS, + PLACEHOLDER_PREFIX, + dump_db, + gcu_add_aggregate, + gcu_add_placeholder_aggregate, + gcu_remove_aggregate, + verify_bgp_aggregate_consistence, + verify_bgp_aggregate_cleanup, +) + +from tests.common.config_reload import config_reload +from tests.common.gcu_utils import create_checkpoint, rollback_or_reload, delete_checkpoint +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.dut_utils import is_virtual_platform +from tests.common.reboot import reboot +from tests.common.utilities import wait_until + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology("m1"), + pytest.mark.disable_loganalyzer, +] + +# ---- Constants ---- +BGP_SESSION_WAIT_TIMEOUT = 300 +BGP_SESSION_POLL_INTERVAL = 10 + + +# ---- Module-scoped setup/teardown ---- +@pytest.fixture(scope="module", autouse=True) +def setup_teardown(duthost): + """Checkpoint before tests, rollback + re-save config after. + + TC5.2/TC5.3/TC5.5 run 'config save -y' before a disruption. If the test + fails after save but before per-test cleanup, the aggregate persists in the + on-disk config_db.json. Re-saving after rollback ensures the saved config + matches the rolled-back (clean) state. + """ + create_checkpoint(duthost) + + default_aggregates = dump_db(duthost, "CONFIG_DB", BGP_AGGREGATE_ADDRESS) + if not default_aggregates: + gcu_add_placeholder_aggregate(duthost, PLACEHOLDER_PREFIX) + + yield + + try: + rollback_or_reload(duthost, fail_on_rollback_error=False) + duthost.shell("sudo config save -y") + finally: + delete_checkpoint(duthost) + + +# ---- Fixtures ---- +@pytest.fixture(scope="module") +def bgp_neighbors(duthosts, rand_one_dut_hostname): + """Return list of BGP neighbor IPs for session-state polling after disruptions.""" + duthost = duthosts[rand_one_dut_hostname] + config_facts = duthost.config_facts(host=duthost.hostname, source="running")["ansible_facts"] + return list(config_facts.get("BGP_NEIGHBOR", {}).keys()) + + +# ---- Helpers ---- +def wait_for_bgp_sessions(duthost, bgp_neighbors): + """Block until all BGP sessions reach Established state.""" + pytest_assert( + wait_until(BGP_SESSION_WAIT_TIMEOUT, BGP_SESSION_POLL_INTERVAL, 0, + duthost.check_bgp_session_state, bgp_neighbors), + "Not all BGP sessions re-established within timeout" + ) + + +def verify_config_db_aggregate(duthost, cfg): + """Lightweight pre-disruption check: only verify CONFIG_DB has the aggregate. + + GCU writes CONFIG_DB synchronously, so this always succeeds immediately. + STATE_DB and FRR depend on bgpcfgd which may be stale after a previous + test session's rollback; those are verified post-disruption instead. + """ + config_db = dump_db(duthost, "CONFIG_DB", BGP_AGGREGATE_ADDRESS) + pytest_assert(cfg.prefix in config_db, f"Aggregate row {cfg.prefix} not found in CONFIG_DB") + pytest_assert( + config_db[cfg.prefix].get("bbr-required") == ("true" if cfg.bbr_required else "false"), + "bbr-required flag mismatch in CONFIG_DB", + ) + pytest_assert( + config_db[cfg.prefix].get("summary-only") == ("true" if cfg.summary_only else "false"), + "summary-only flag mismatch in CONFIG_DB", + ) + pytest_assert( + config_db[cfg.prefix].get("as-set") == ("true" if cfg.as_set else "false"), + "as-set flag mismatch in CONFIG_DB", + ) + + +def _state_db_has_prefix(duthost, prefix): + """Return True when bgpcfgd has populated STATE_DB for the given prefix.""" + state_db = dump_db(duthost, "STATE_DB", BGP_AGGREGATE_ADDRESS) + return prefix in state_db + + +def wait_for_aggregate_state(duthost, prefix, timeout=60): + """Poll until bgpcfgd populates STATE_DB for the aggregate prefix. + + GCU writes CONFIG_DB synchronously, but bgpcfgd processes the change + and writes STATE_DB asynchronously. This helper bridges the race after + a disruption that restarts bgpcfgd. + """ + pytest_assert( + wait_until(timeout, 5, 0, _state_db_has_prefix, duthost, prefix), + f"STATE_DB entry for {prefix} not populated within {timeout}s" + ) + + +# =========================================================================== +# Test Case 5.1 — BGP container restart +# =========================================================================== +def test_aggregate_persists_bgp_container_restart( + duthosts, rand_one_dut_hostname, bgp_neighbors +): + """ + TC 5.1: Aggregate address config survives BGP container restart. + + Steps: + 1. Add aggregate via GCU, verify CONFIG_DB. + 2. Restart BGP container. + 3. Wait for BGP sessions to re-establish. + 4. Verify full consistency (CONFIG_DB + STATE_DB + FRR). + """ + duthost = duthosts[rand_one_dut_hostname] + bbr_enabled = is_bbr_enabled(duthost) + cfg = AggregateCfg(prefix=AGGR_V4, bbr_required=False, summary_only=False, as_set=False) + + try: + gcu_add_aggregate(duthost, cfg) + verify_config_db_aggregate(duthost, cfg) + + duthost.shell("sudo systemctl restart bgp") + wait_for_bgp_sessions(duthost, bgp_neighbors) + + wait_for_aggregate_state(duthost, cfg.prefix) + verify_bgp_aggregate_consistence(duthost, bbr_enabled, cfg) + finally: + try: + gcu_remove_aggregate(duthost, cfg.prefix) + except Exception: + logger.warning("Cleanup: failed to remove aggregate %s, will be recovered by rollback", cfg.prefix) + verify_bgp_aggregate_cleanup(duthost, cfg.prefix) + + +# =========================================================================== +# Test Case 5.2 — Config reload +# =========================================================================== +def test_aggregate_persists_config_reload( + duthosts, rand_one_dut_hostname, bgp_neighbors +): + """ + TC 5.2: Aggregate address config survives config reload. + + Steps: + 1. Add aggregate with summary-only=true via GCU, verify CONFIG_DB. + 2. Save config and execute config reload. + 3. Wait for BGP sessions. + 4. Verify full consistency after reload. + """ + duthost = duthosts[rand_one_dut_hostname] + bbr_enabled = is_bbr_enabled(duthost) + cfg = AggregateCfg(prefix=AGGR_V4, bbr_required=False, summary_only=True, as_set=False) + + try: + gcu_add_aggregate(duthost, cfg) + verify_config_db_aggregate(duthost, cfg) + + duthost.shell("sudo config save -y") + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) + wait_for_bgp_sessions(duthost, bgp_neighbors) + + wait_for_aggregate_state(duthost, cfg.prefix) + verify_bgp_aggregate_consistence(duthost, bbr_enabled, cfg) + finally: + try: + gcu_remove_aggregate(duthost, cfg.prefix) + except Exception: + logger.warning("Cleanup: failed to remove aggregate %s, will be recovered by rollback", cfg.prefix) + verify_bgp_aggregate_cleanup(duthost, cfg.prefix) + + +# =========================================================================== +# Test Case 5.3 — Config save and cold reboot +# =========================================================================== +def test_aggregate_persists_config_save_and_reboot( + duthosts, rand_one_dut_hostname, localhost, bgp_neighbors +): + """ + TC 5.3: Aggregate address config survives config save followed by cold reboot. + + Steps: + 1. Add IPv6 aggregate via GCU, verify CONFIG_DB. + 2. Save config and cold reboot. + 3. Wait for BGP sessions. + 4. Verify full consistency after reboot. + """ + duthost = duthosts[rand_one_dut_hostname] + bbr_enabled = is_bbr_enabled(duthost) + cfg = AggregateCfg(prefix=AGGR_V6, bbr_required=False, summary_only=False, as_set=False) + + try: + gcu_add_aggregate(duthost, cfg) + verify_config_db_aggregate(duthost, cfg) + + duthost.shell("sudo config save -y") + reboot(duthost, localhost, reboot_type="cold", safe_reboot=True, + check_intf_up_ports=True, wait_for_bgp=True) + wait_for_bgp_sessions(duthost, bgp_neighbors) + + wait_for_aggregate_state(duthost, cfg.prefix) + verify_bgp_aggregate_consistence(duthost, bbr_enabled, cfg) + finally: + try: + gcu_remove_aggregate(duthost, cfg.prefix) + except Exception: + logger.warning("Cleanup: failed to remove aggregate %s, will be recovered by rollback", cfg.prefix) + verify_bgp_aggregate_cleanup(duthost, cfg.prefix) + + +# =========================================================================== +# Test Case 5.4 — BGP container restart with BBR-required inactive aggregate +# =========================================================================== +def test_aggregate_bbr_required_inactive_persists_bgp_restart( + duthosts, rand_one_dut_hostname, bgp_neighbors +): + """ + TC 5.4: BBR-required aggregate stays inactive after BGP restart when BBR + is disabled, then activates once BBR is enabled. + + Steps: + 1. Disable BBR. + 2. Add BBR-required aggregate, verify CONFIG_DB. + 3. Restart BGP container — verify still inactive. + 4. Enable BBR — verify aggregate now active. + """ + duthost = duthosts[rand_one_dut_hostname] + bbr_supported, bbr_default_state = get_bbr_default_state(duthost) + if not bbr_supported: + pytest.skip("BGP BBR is not supported") + + cfg = AggregateCfg(prefix=AGGR_V4, bbr_required=True, summary_only=False, as_set=False) + + try: + # Ensure BBR is disabled + if bbr_default_state == "enabled": + config_bbr_by_gcu(duthost, "disabled") + + gcu_add_aggregate(duthost, cfg) + verify_config_db_aggregate(duthost, cfg) + + duthost.shell("sudo systemctl restart bgp") + wait_for_bgp_sessions(duthost, bgp_neighbors) + + wait_for_aggregate_state(duthost, cfg.prefix) + verify_bgp_aggregate_consistence(duthost, False, cfg) + + # Enable BBR — aggregate should become active + config_bbr_by_gcu(duthost, "enabled") + verify_bgp_aggregate_consistence(duthost, True, cfg) + finally: + try: + gcu_remove_aggregate(duthost, cfg.prefix) + except Exception: + logger.warning("Cleanup: failed to remove aggregate %s, will be recovered by rollback", cfg.prefix) + verify_bgp_aggregate_cleanup(duthost, cfg.prefix) + # Restore BBR to original state + try: + config_bbr_by_gcu(duthost, bbr_default_state) + except Exception: + logger.warning("Cleanup: failed to restore BBR state to %s", bbr_default_state) + + +# =========================================================================== +# Test Case 5.5 — Warm reboot +# =========================================================================== +def test_aggregate_persists_warm_reboot( + duthosts, rand_one_dut_hostname, localhost, bgp_neighbors +): + """ + TC 5.5: Aggregate address config survives warm reboot. + + Steps: + 1. Add aggregate via GCU, save config, verify CONFIG_DB. + 2. Warm reboot. + 3. Wait for BGP sessions. + 4. Verify full consistency after warm reboot. + """ + duthost = duthosts[rand_one_dut_hostname] + bbr_enabled = is_bbr_enabled(duthost) + cfg = AggregateCfg(prefix=AGGR_V4, bbr_required=False, summary_only=False, as_set=False) + + # On KVM/VS the warm-reboot script's 1 s docker-exec health check is too + # tight, causing fpmsyncd/orchagent crashes. Apply the same timeout bump + # that AdvancedReboot uses (tests/common/fixtures/advanced_reboot.py). + if is_virtual_platform(duthost): + warmboot_script = duthost.shell("which warm-reboot")["stdout"].strip() + cmd_format = "sed -i 's/{}/{}/' {}" + original_line = 'timeout 1s docker exec $container echo "success"' + replaced_line = 'timeout 5s docker exec $container echo "success"' + duthost.shell(cmd_format.format(original_line, replaced_line, warmboot_script)) + + try: + gcu_add_aggregate(duthost, cfg) + duthost.shell("sudo config save -y") + verify_config_db_aggregate(duthost, cfg) + + reboot(duthost, localhost, reboot_type="warm", safe_reboot=True, + check_intf_up_ports=True, wait_for_bgp=True, + wait_warmboot_finalizer=True) + wait_for_bgp_sessions(duthost, bgp_neighbors) + + wait_for_aggregate_state(duthost, cfg.prefix) + verify_bgp_aggregate_consistence(duthost, bbr_enabled, cfg) + finally: + try: + gcu_remove_aggregate(duthost, cfg.prefix) + except Exception: + logger.warning("Cleanup: failed to remove aggregate %s, will be recovered by rollback", cfg.prefix) + verify_bgp_aggregate_cleanup(duthost, cfg.prefix)