diff --git a/tests/common/macsec/macsec_helper.py b/tests/common/macsec/macsec_helper.py index 55b90656cd5..402db8eadf6 100644 --- a/tests/common/macsec/macsec_helper.py +++ b/tests/common/macsec/macsec_helper.py @@ -181,16 +181,23 @@ def __check_appl_db(duthost, dut_ctrl_port_name, nbrhost, nbr_ctrl_port_name, po assert dut_ingress_sc_table and nbr_ingress_sc_table assert dut_egress_sc_table and nbr_egress_sc_table - # CHeck MACsec SA Table - assert int(dut_egress_sc_table["encoding_an"]) in dut_egress_sa_table - assert int(nbr_egress_sc_table["encoding_an"]) in nbr_egress_sa_table - for egress_sas, ingress_sas in \ - ((dut_egress_sa_table, nbr_ingress_sa_table), (nbr_egress_sa_table, dut_ingress_sa_table)): - for an, sa in list(egress_sas.items()): - assert an in ingress_sas - assert sa["sak"] == ingress_sas[an]["sak"] - assert sa["auth_key"] == ingress_sas[an]["auth_key"] - assert sa["next_pn"] >= ingress_sas[an]["lowest_acceptable_pn"] + # Check MACsec SA Table. Only the active encoding_an SA needs to be + # consistent between egress and peer ingress. Non-encoding ANs may linger + # in APPL_DB after a dirty container kill (macsecmgrd had no chance to + # clean them up), while the peer correctly only re-installs the current AN + # after MKA re-establishes. Checking all ANs would cause spurious + # convergence failures in the post-dirty-kill window. + for egress_sc, egress_sa_table, peer_ingress_sa_table in \ + ((dut_egress_sc_table, dut_egress_sa_table, nbr_ingress_sa_table), + (nbr_egress_sc_table, nbr_egress_sa_table, dut_ingress_sa_table)): + encoding_an = int(egress_sc["encoding_an"]) + assert encoding_an in egress_sa_table + assert encoding_an in peer_ingress_sa_table + egress_sa = egress_sa_table[encoding_an] + ingress_sa = peer_ingress_sa_table[encoding_an] + assert egress_sa["sak"] == ingress_sa["sak"] + assert egress_sa["auth_key"] == ingress_sa["auth_key"] + assert egress_sa["next_pn"] >= ingress_sa["lowest_acceptable_pn"] def check_appl_db(duthost, ctrl_links, policy, cipher_suite, send_sci): diff --git a/tests/common/macsec/recovery_helpers.py b/tests/common/macsec/recovery_helpers.py new file mode 100644 index 00000000000..69e6ac8c377 --- /dev/null +++ b/tests/common/macsec/recovery_helpers.py @@ -0,0 +1,394 @@ +import logging +import time + +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.dut_utils import ( + restart_service_with_startlimit_guard, + is_container_running, + is_hitting_start_limit, +) +from tests.common.macsec.macsec_helper import ( + check_appl_db, + get_sci, + getns_prefix, +) + + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Tunables +# --------------------------------------------------------------------------- +# Seconds to wait after SIGKILL before polling for the container to be up. +KILL_SETTLE_SECONDS = 5 +# Maximum seconds to wait for the macsec container to respawn. +CONTAINER_UP_TIMEOUT = 120 +# Maximum seconds to wait for MKA (re-)convergence. +MKA_CONVERGE_TIMEOUT = 300 +MKA_CONVERGE_INTERVAL = 6 +MKA_CONVERGE_DELAY = 12 + + +# --------------------------------------------------------------------------- +# Disruption primitives +# --------------------------------------------------------------------------- + +def graceful_restart_macsec(duthost): + """`systemctl restart macsec` via the startlimit-aware helper.""" + logger.info("Graceful restart of macsec on %s", duthost.hostname) + restart_service_with_startlimit_guard( + duthost, "macsec", + backoff_seconds=35, + verify_timeout=CONTAINER_UP_TIMEOUT, + ) + + +def dirty_kill_macsec_container(duthost): + """`docker kill -s 9 macsec` — bypasses macsecmgrd's per-port disable loop.""" + logger.info("Sending SIGKILL to macsec container on %s", duthost.hostname) + duthost.shell("docker kill -s 9 macsec", module_ignore_errors=False) + + +def dirty_kill_macsecmgrd(duthost, signal=9): + """ + Send a signal to macsecmgrd inside the macsec container. + + signal=9 (SIGKILL) skips graceful shutdown; signal=6 (SIGABRT) generates a + core dump. supervisord inside the container respawns macsecmgrd; the + container itself stays up, as do the per-port wpa_supplicant processes + and their UNIX control sockets. + """ + logger.info("Sending signal %d to macsecmgrd inside macsec container on %s", + signal, duthost.hostname) + duthost.shell( + "docker exec macsec pkill -{} -x macsecmgrd".format(signal), + module_ignore_errors=False, + ) + + +def dirty_kill_wpa_supplicant(duthost, port_name): + """ + SIGKILL the wpa_supplicant process bound to one MACsec port. + + The wpa_supplicant command line includes the per-port control socket + path (/var/run/Ethernet), so we use that to scope the pkill to one + instance. macsecmgrd respawns it; other ports' wpa_supplicants are + untouched. + """ + logger.info("SIGKILL wpa_supplicant for %s on %s", + port_name, duthost.hostname) + duthost.shell( + "docker exec macsec pkill -9 -f '/var/run/{}'".format(port_name), + module_ignore_errors=False, + ) + + +# --------------------------------------------------------------------------- +# Recovery waits +# --------------------------------------------------------------------------- + +def wait_for_macsec_container(duthost): + """ + Wait for systemd to *auto-respawn* the macsec container after a dirty + kill. Deliberately does NOT issue `systemctl restart`: the macsec + service has a Restart= policy, so after SIGKILL systemd brings the + container back on its own. A `systemctl restart` here would graceful- + stop the freshly auto-respawned container first, giving macsecmgrd a + clean per-port teardown that wipes orchagent's stale SA state — which + converts the dirty restart into a graceful one. + + Fallback: if the container hasn't come back within CONTAINER_UP_TIMEOUT + (e.g. rapid repeated kills tripped systemd's StartLimitHit so auto- + respawn is suppressed), clear the failure counter and `start` it — a + start, never a restart, so a still-running container is never gracefully + stopped. + """ + time.sleep(KILL_SETTLE_SECONDS) + + if wait_until(CONTAINER_UP_TIMEOUT, 2, 0, + is_container_running, duthost, "macsec"): + logger.info("macsec container auto-respawned on %s", duthost.hostname) + return + + logger.warning( + "macsec did not auto-respawn on %s (StartLimitHit=%s); " + "clearing failure counter and starting", + duthost.hostname, is_hitting_start_limit(duthost, "macsec")) + duthost.shell("sudo systemctl reset-failed macsec.service", + module_ignore_errors=True) + duthost.shell("sudo systemctl start macsec.service", + module_ignore_errors=True) + pytest_assert( + wait_until(CONTAINER_UP_TIMEOUT, 2, 0, + is_container_running, duthost, "macsec"), + "macsec container did not come up after dirty kill + fallback start") + logger.info("macsec container started via fallback on %s", duthost.hostname) + + +def wait_for_mka_converged(duthost, ctrl_links, policy, cipher_suite, send_sci): + """ + Poll APPL_DB until check_appl_db reports MKA converged on every ctrl_link. + Returns True on success, False on timeout. + """ + return wait_until( + MKA_CONVERGE_TIMEOUT, + MKA_CONVERGE_INTERVAL, + MKA_CONVERGE_DELAY, + check_appl_db, + duthost, ctrl_links, policy, cipher_suite, send_sci, + ) + + +# --------------------------------------------------------------------------- +# APPL_DB snapshots / invariants +# --------------------------------------------------------------------------- + +def _get_appl_db_sa_sak(duthost, port_name, sci, an, egress=True): + """Return the SAK from APPL_DB for the given (port, sci, an), or None.""" + table = "MACSEC_EGRESS_SA_TABLE" if egress else "MACSEC_INGRESS_SA_TABLE" + ns_prefix = getns_prefix(duthost, port_name) + cmd = "sonic-db-cli {} APPL_DB HGET '{}:{}:{}:{}' sak".format( + ns_prefix, table, port_name, sci, an) + result = duthost.shell(cmd, module_ignore_errors=True) + sak = result.get("stdout", "").strip() + return sak if sak else None + + +def snapshot_appl_db_saks(duthost, ctrl_links): + """ + Snapshot every (port, sci, an, direction) -> sak currently in APPL_DB + across all macsec ctrl_links. Useful as a before/after pivot for tests + that need to detect SAK churn. + """ + saks = {} + for port_name, nbr in ctrl_links.items(): + host_sci = get_sci(duthost.get_dut_iface_mac(port_name)) + peer_sci = get_sci(nbr["host"].get_dut_iface_mac(nbr["port"])) + for an in range(4): + v = _get_appl_db_sa_sak(duthost, port_name, host_sci, an, egress=True) + if v: + saks[(port_name, host_sci, an, "egress")] = v + v = _get_appl_db_sa_sak(duthost, port_name, peer_sci, an, egress=False) + if v: + saks[(port_name, peer_sci, an, "ingress")] = v + return saks + + +def _asic_db_macsec_saks(duthost, ctrl_links): + """ + Collect the set of SAKs currently programmed in ASIC_DB + (SAI_OBJECT_TYPE_MACSEC_SA.SAI_MACSEC_SA_ATTR_SAK) across the + namespaces that host the ctrl_link ports. + + ASIC_DB mirrors what syncd actually programmed into SAI/the chip, so + it is the source of truth for the hardware SAK. SAKs are returned + upper-cased for case-insensitive comparison against APPL_DB. + """ + ns_prefixes = set(getns_prefix(duthost, port_name) for port_name in ctrl_links) + saks = set() + for ns_prefix in ns_prefixes: + keys = duthost.shell( + "sonic-db-cli {} ASIC_DB KEYS " + "'ASIC_STATE:SAI_OBJECT_TYPE_MACSEC_SA:*'".format(ns_prefix), + module_ignore_errors=True, + ).get("stdout", "").split() + for key in keys: + sak = duthost.shell( + "sonic-db-cli {} ASIC_DB HGET '{}' " + "SAI_MACSEC_SA_ATTR_SAK".format(ns_prefix, key), + module_ignore_errors=True, + ).get("stdout", "").strip() + if sak: + saks.add(sak.upper()) + return saks + + +def assert_appl_db_sak_programmed_in_asic(duthost, ctrl_links): + """ + For every MACsec SA in APPL_DB, assert its SAK is actually present in + ASIC_DB — i.e. the key orchagent advertises was really programmed into + SAI/the chip. + + After a dirty restart wpa renegotiates a new SAK at the same (port, + sci, AN), but the stale SA object survives in orchagent's + MACsecSC::m_sa_ids, so createMACsecSA short-circuits and never + reprograms SAI. SAI_MACSEC_SA_ATTR_SAK is CREATE-ONLY, so the chip + keeps the prior cycle's key while APPL_DB carries the fresh one. + + Raises AssertionError listing every APPL_DB SAK absent from ASIC_DB. + """ + appl_saks = snapshot_appl_db_saks(duthost, ctrl_links) + asic_saks = _asic_db_macsec_saks(duthost, ctrl_links) + + pytest_assert( + asic_saks, + "ASIC_DB has no MACSEC_SA objects at all — cannot validate SAK " + "programming (macsec not converged in hardware?)") + + # SAI_MACSEC_SA_ATTR_SAK is always a 256-bit (64 hex chars) field in + # ASIC_DB regardless of cipher suite. For GCM-AES-128 the APPL_DB SAK + # is 32 hex chars (128 bits); pad it to 64 chars before comparing so + # AES-128 and AES-256 keys both match correctly. + SAK_ASIC_FIELD_LEN = 64 + failures = [] + for (port_name, sci, an, direction), appl_sak in sorted(appl_saks.items()): + normalized = appl_sak.upper().zfill(SAK_ASIC_FIELD_LEN) + if normalized not in asic_saks: + failures.append( + "port={} sci={} an={} dir={}: APPL_DB sak={} is NOT present " + "in ASIC_DB (stale-SAK bug: orchagent left the prior SAK in " + "SAI after re-key)".format( + port_name, sci, an, direction, appl_sak)) + + if failures: + raise AssertionError( + "APPL_DB->ASIC_DB SAK mismatch ({} entry/entries); ASIC_DB holds " + "{} distinct SAK(s):\n{}".format( + len(failures), len(asic_saks), + "\n".join(" * " + f for f in failures))) + + +# --------------------------------------------------------------------------- +# Encoding-AN (stale-AN) helpers +# --------------------------------------------------------------------------- + +def get_egress_encoding_ans(duthost, ctrl_links): + """ + Return {(port, host_sci): encoding_an} read from APPL_DB + MACSEC_EGRESS_SC_TABLE for every ctrl_link. encoding_an is the AN the + egress SC is currently encrypting with. + """ + ans = {} + for port_name in ctrl_links: + host_sci = get_sci(duthost.get_dut_iface_mac(port_name)) + ns_prefix = getns_prefix(duthost, port_name) + v = duthost.shell( + "sonic-db-cli {} APPL_DB HGET 'MACSEC_EGRESS_SC_TABLE:{}:{}' " + "encoding_an".format(ns_prefix, port_name, host_sci), + module_ignore_errors=True, + ).get("stdout", "").strip() + if v != "": + ans[(port_name, host_sci)] = v + return ans + + +def advance_egress_encoding_an(duthost, profile_name, ctrl_links, + policy, cipher_suite, send_sci, + rekey_period=20, advance_timeout=240): + """ + Force the egress encoding AN past 0 by briefly enabling a short MKA rekey + period, so a subsequent dirty restart crosses an AN boundary — the + precondition for the stale-AN bug (the fix's sweep only runs when + new_an != m_encoding_an). + + Sets rekey_period on the DUT MACSEC_PROFILE, graceful-restarts macsec to + apply it (the session starts at AN=0), waits for MKA to converge, then + polls until at least one egress SC's encoding_an advances to a non-zero + value (i.e. one rekey has happened). Leaves rekey_period set; the caller + is responsible for restoring it (typically to 0 while macsec is killed so + the post-restart session is stable). + + Returns the {(port, sci): encoding_an} map observed after the advance. + """ + duthost.shell( + "sonic-db-cli CONFIG_DB HSET 'MACSEC_PROFILE|{}' rekey_period {}".format( + profile_name, rekey_period), + module_ignore_errors=False, + ) + graceful_restart_macsec(duthost) + pytest_assert( + wait_for_mka_converged(duthost, ctrl_links, policy, cipher_suite, send_sci), + "MKA did not converge after enabling rekey_period={}".format(rekey_period)) + + def _an_advanced(): + ans = get_egress_encoding_ans(duthost, ctrl_links) + return ans and all(int(an) >= 1 for an in ans.values()) + + pytest_assert( + wait_until(advance_timeout, 5, 0, _an_advanced), + "egress encoding_an did not advance to >=1 within {}s with " + "rekey_period={} (cannot exercise stale-AN path)".format( + advance_timeout, rekey_period)) + return get_egress_encoding_ans(duthost, ctrl_links) + + +def set_rekey_period(duthost, profile_name, rekey_period): + """Set rekey_period on the DUT MACSEC_PROFILE (does not restart macsec).""" + duthost.shell( + "sonic-db-cli CONFIG_DB HSET 'MACSEC_PROFILE|{}' rekey_period {}".format( + profile_name, rekey_period), + module_ignore_errors=False, + ) + + +def _asic_db_egress_sa_ans_by_sc(duthost, ctrl_links): + """ + Return {sc_oid: [an, ...]} for every EGRESS MACsec Secure Channel in + ASIC_DB, listing the ANs of the egress SAs installed under each SC. + + A single shell loop on the DUT keeps this to one round-trip per + namespace. Output lines are 'EGRESS||' per egress SA. + """ + ns_prefixes = set(getns_prefix(duthost, port_name) for port_name in ctrl_links) + sc_to_ans = {} + for ns_prefix in ns_prefixes: + script = ( + "for k in $(sonic-db-cli {ns} ASIC_DB KEYS " + "'ASIC_STATE:SAI_OBJECT_TYPE_MACSEC_SA:*' 2>/dev/null); do " + "d=$(sonic-db-cli {ns} ASIC_DB HGET \"$k\" " + "SAI_MACSEC_SA_ATTR_MACSEC_DIRECTION); " + "sc=$(sonic-db-cli {ns} ASIC_DB HGET \"$k\" " + "SAI_MACSEC_SA_ATTR_SC_ID); " + "an=$(sonic-db-cli {ns} ASIC_DB HGET \"$k\" " + "SAI_MACSEC_SA_ATTR_AN); " + "echo \"$d|$sc|$an\"; done" + ).format(ns=ns_prefix) + out = duthost.shell(script, module_ignore_errors=True).get("stdout", "") + for line in out.splitlines(): + parts = line.split("|") + if len(parts) != 3: + continue + direction, sc_oid, an = parts + if "EGRESS" not in direction or not sc_oid: + continue + sc_to_ans.setdefault(sc_oid, []).append(an) + return sc_to_ans + + +def assert_one_egress_sa_per_sc(duthost, ctrl_links): + """ + After recovery, every egress Secure Channel must have exactly one SA in + ASIC_DB. + + Stale-AN bug (SC side): a dirty restart kills wpa between + enable_transmit_sa(new_an) and delete_transmit_sa(old_an). The prior + AN's SA survives in orchagent's sc.m_sa_ids and in SAI. When the fresh + session rekeys onto a different AN, setEncodingAN updates m_encoding_an + but (unfixed) never sweeps the leaked old-AN SA, so the egress SC ends + up with two installed SAs and the chip encrypts with the stale one while + the peer's ingress SC has no SA at that AN -> IN_PKTS_NOT_USING_SA, LACP + down. The fix sweeps non-current ANs, restoring one-SA-per-SC. + + Raises AssertionError listing every egress SC carrying more than one SA. + """ + sc_to_ans = _asic_db_egress_sa_ans_by_sc(duthost, ctrl_links) + + pytest_assert( + sc_to_ans, + "ASIC_DB has no egress MACSEC_SA objects — cannot validate the " + "one-SA-per-SC invariant (macsec not converged in hardware?)") + + failures = [] + for sc_oid, ans in sorted(sc_to_ans.items()): + if len(ans) != 1: + failures.append( + "egress SC {} has {} SAs at AN(s) {} (expected exactly 1); " + "stale-AN bug left the pre-restart SA installed alongside the " + "post-rekey one".format(sc_oid, len(ans), sorted(ans))) + + if failures: + raise AssertionError( + "egress SC one-SA invariant violated ({} SC(s)):\n{}".format( + len(failures), "\n".join(" * " + f for f in failures))) diff --git a/tests/macsec/test_macsec_recovery.py b/tests/macsec/test_macsec_recovery.py new file mode 100644 index 00000000000..5216f89c71b --- /dev/null +++ b/tests/macsec/test_macsec_recovery.py @@ -0,0 +1,248 @@ +import logging +import pytest + +from tests.common.macsec.recovery_helpers import ( + advance_egress_encoding_an, + assert_appl_db_sak_programmed_in_asic, + assert_one_egress_sa_per_sc, + dirty_kill_macsec_container, + get_egress_encoding_ans, + graceful_restart_macsec, + set_rekey_period, + snapshot_appl_db_saks, + wait_for_macsec_container, + wait_for_mka_converged, + MKA_CONVERGE_TIMEOUT, +) + + +logger = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def macsec_loganalyzer_ignore(loganalyzer, macsec_duthost): + """ + Suppress error messages that are expected during dirty-restart recovery + and are not indicative of real faults: + + - DNX SAI 'Invalid object ID': syncd polls SA attributes after orchagent + deletes a stale SA; the DNX platform logs this transiently. + - 'KaY: The key server is not in my live peers list': wpa_supplicant logs + this while MKA re-negotiates after the macsec container is killed. + - 'e1000 ... Reset adapter': KVM virtual NIC resets when the macsec + container is SIGKILL'd; this is a VTB environment artifact. + """ + if loganalyzer and macsec_duthost: + loganalyzer[macsec_duthost.hostname].ignore_regex.extend([ + r".*SAI_API_MACSEC:brcm_sai_dnx_get_macsec_sa_attribute.*Invalid object ID.*", + r".*macsec\d*#wpa_supplicant.*KaY: The key server is not in my live peers list.*", + r".*kernel:.*e1000.*Reset adapter.*", + ]) + + +pytestmark = [ + pytest.mark.macsec_required, + pytest.mark.topology("t0", "t2", "t0-sonic", "any"), +] + + +@pytest.fixture +def force_dut_key_server(macsec_duthost, profile_name, ctrl_links, + policy, cipher_suite, send_sci): + """ + Lower the DUT's MACSEC_PROFILE priority below the peer's so the DUT + deterministically wins MKA Key Server election on every ctrl_link. + + Required by the stale-AN test: the egress encoding AN must differ + between the surviving (pre-kill) SC and the fresh post-restart session. + With the DUT as key server, a fresh restart resets its egress AN to 0, + so advancing the AN to >=1 before the kill guarantees the asymmetry the + bug needs. (Not needed for the stale-SAK test, where the AN is fixed.) + + Setup: HSET priority 0; graceful-restart; wait for re-converge. + Teardown: restore original priority AND rekey_period, then graceful- + restart. rekey_period is restored here (not just in the test body) so + that an advance-timeout error mid-test can't leak a non-zero rekey_period + into the next profile's run. + """ + duthost = macsec_duthost + + orig_priority = duthost.shell( + "sonic-db-cli CONFIG_DB HGET 'MACSEC_PROFILE|{}' priority".format( + profile_name), + module_ignore_errors=True, + )["stdout"].strip() or "64" + orig_rekey = duthost.shell( + "sonic-db-cli CONFIG_DB HGET 'MACSEC_PROFILE|{}' rekey_period".format( + profile_name), + module_ignore_errors=True, + )["stdout"].strip() or "0" + logger.info("force_dut_key_server: original priority=%s rekey_period=%s, " + "forcing priority to 0", orig_priority, orig_rekey) + + duthost.shell( + "sonic-db-cli CONFIG_DB HSET 'MACSEC_PROFILE|{}' priority 0".format( + profile_name), + module_ignore_errors=False, + ) + graceful_restart_macsec(duthost) + assert wait_for_mka_converged( + duthost, ctrl_links, policy, cipher_suite, send_sci), \ + "MKA did not converge after forcing DUT KS priority" + + yield + + logger.info("force_dut_key_server teardown: restoring priority=%s " + "rekey_period=%s", orig_priority, orig_rekey) + duthost.shell( + "sonic-db-cli CONFIG_DB HSET 'MACSEC_PROFILE|{}' priority {} " + "rekey_period {}".format(profile_name, orig_priority, orig_rekey), + module_ignore_errors=False, + ) + graceful_restart_macsec(duthost) + + +@pytest.mark.backstop +def test_dirty_container_kill_preserves_sak_consistency( + macsec_duthost, ctrl_links, policy, cipher_suite, send_sci, + force_dut_key_server): + """ + SIGKILL the macsec container so macsecmgrd has no chance to gracefully + disable per-port MACsec. After respawn + MKA re-convergence, the SAK + advertised in APPL_DB must actually be programmed into the ASIC + (ASIC_DB SAI_OBJECT_TYPE_MACSEC_SA.SAI_MACSEC_SA_ATTR_SAK). + + The stale-SAK class of bug shows up here: on respawn wpa + renegotiates a new SAK at the same (port, sci, AN), but the pre-kill SA + object survives in orchagent's MACsecSC::m_sa_ids, so createMACsecSA + short-circuits and never reprograms SAI. SAI_MACSEC_SA_ATTR_SAK is + CREATE-ONLY, so the ASIC keeps the prior cycle's key while APPL_DB + carries the fresh one. This is deterministic per dirty kill; the + ASIC_DB comparison is what surfaces it (`show macsec` reads APPL_DB and + cannot). + + force_dut_key_server is required so the DUT's freshly-restarted + wpa_supplicant acts as MKA key server and generates a new SAK after + restart. Without this, a peer KS may redistribute the same SAK (no + change detected), making step 6b vacuous. + """ + duthost = macsec_duthost + + logger.info("Step 1: verifying initial MKA convergence on %s", + duthost.hostname) + assert wait_for_mka_converged( + duthost, ctrl_links, policy, cipher_suite, send_sci), \ + "MKA did not converge before dirty-restart test" + + logger.info("Step 2: snapshotting pre-kill SAK values") + pre_kill_saks = snapshot_appl_db_saks(duthost, ctrl_links) + logger.info("Pre-kill SAK snapshot has %d entries", len(pre_kill_saks)) + + logger.info("Step 3: dirty-killing macsec container on %s", + duthost.hostname) + dirty_kill_macsec_container(duthost) + + logger.info("Step 4: waiting for macsec container to respawn") + wait_for_macsec_container(duthost) + + logger.info("Step 5: waiting for MKA re-convergence (timeout=%ds)", + MKA_CONVERGE_TIMEOUT) + assert wait_for_mka_converged( + duthost, ctrl_links, policy, cipher_suite, send_sci), \ + "MKA did not re-converge within {}s after dirty restart".format( + MKA_CONVERGE_TIMEOUT) + + logger.info("Step 6a: confirming MKA actually re-keyed (silent-pass guard)") + post_recovery_saks = snapshot_appl_db_saks(duthost, ctrl_links) + changed = [ + k for k, v in pre_kill_saks.items() + if k in post_recovery_saks and post_recovery_saks[k] != v + ] + assert changed, ( + "MKA did not re-key after dirty restart: pre-kill and post-recovery " + "SAKs identical on all {} entry/entries. The SAK consistency check " + "below would be vacuous — failing now to catch this silent-pass mode." + ).format(len(pre_kill_saks)) + logger.info("MKA re-keyed on %d/%d SA entries", + len(changed), len(pre_kill_saks)) + + logger.info("Step 6b: verifying APPL_DB SAKs are programmed into ASIC_DB") + assert_appl_db_sak_programmed_in_asic(duthost, ctrl_links) + logger.info("APPL_DB SAKs confirmed in ASIC_DB on all %d ctrl_link ports", + len(ctrl_links)) + + +@pytest.mark.backstop +def test_dirty_container_kill_recovers_encoding_an( + macsec_duthost, profile_name, ctrl_links, policy, cipher_suite, + send_sci, force_dut_key_server): + """ + Stale-AN regression guard. Distinct from the stale-SAK case: here the + egress encoding AN must CHANGE across the dirty restart, because the fix + (and thus the bug) is gated on `new_an != sc.m_encoding_an` in + setEncodingAN — an unchanged AN hits the idempotent early-return and is + never swept. + + Recipe: + 1. DUT is key server (fixture), so its egress AN is locally controlled + and a fresh restart resets it to AN=0. + 2. Advance the egress encoding AN to >=1 via a brief rekey. + 3. Dirty kill: the surviving egress SC keeps m_encoding_an>=1 and its + AN>=1 SA in sc.m_sa_ids / SAI. + 4. Restore rekey_period=0 while down so the respawned DUT-KS session is + stable at AN=0. + 5. After recovery the fresh session sets encoding_an=0; with the + surviving AN>=1 SA still installed, an unfixed orchagent leaves two + egress SAs on the SC (the chip then encrypts with the stale one and + the peer drops every frame). + + Detector: ASIC_DB must show exactly one egress SA per SC. + + Scoped to the 128/256 GCM-AES-XPN profiles — the two most common in + production. The bug is in AN/SC bookkeeping, not crypto, so it is + cipher-suite-independent; running every profile only multiplied runtime + and exposed cumulative testbed degradation across the long sweep. + """ + if profile_name not in ("128_XPN", "256_XPN"): + pytest.skip( + "stale-AN test runs only on 128_XPN/256_XPN (most common in " + "production); the bug is cipher-suite-independent") + + duthost = macsec_duthost + + logger.info("Step 1: verifying initial MKA convergence on %s", + duthost.hostname) + assert wait_for_mka_converged( + duthost, ctrl_links, policy, cipher_suite, send_sci), \ + "MKA did not converge before stale-AN test" + + logger.info("Step 2: advancing egress encoding AN past 0 via brief rekey") + advanced = advance_egress_encoding_an( + duthost, profile_name, ctrl_links, policy, cipher_suite, send_sci) + logger.info("Pre-kill egress encoding_an: %s", advanced) + + logger.info("Step 3: dirty-killing macsec container on %s", + duthost.hostname) + dirty_kill_macsec_container(duthost) + + logger.info("Step 4: restoring rekey_period=0 so the respawned session " + "is stable at AN=0") + set_rekey_period(duthost, profile_name, 0) + + logger.info("Step 5: waiting for macsec container to respawn") + wait_for_macsec_container(duthost) + + logger.info("Step 6: waiting for MKA re-convergence (timeout=%ds)", + MKA_CONVERGE_TIMEOUT) + assert wait_for_mka_converged( + duthost, ctrl_links, policy, cipher_suite, send_sci), \ + "MKA did not re-converge within {}s after dirty restart".format( + MKA_CONVERGE_TIMEOUT) + + post = get_egress_encoding_ans(duthost, ctrl_links) + logger.info("Post-recovery egress encoding_an: %s", post) + + logger.info("Step 7: verifying exactly one egress SA per SC in ASIC_DB") + assert_one_egress_sa_per_sc(duthost, ctrl_links) + logger.info("One-SA-per-SC invariant holds on all %d ctrl_link ports", + len(ctrl_links))