diff --git a/.azure-pipelines/pr_test_scripts.yaml b/.azure-pipelines/pr_test_scripts.yaml index bcb150ab6aa..219d6fb1252 100644 --- a/.azure-pipelines/pr_test_scripts.yaml +++ b/.azure-pipelines/pr_test_scripts.yaml @@ -678,6 +678,7 @@ t1-lag-vpp: - bgp/test_traffic_shift_sup.py - crm/test_crm.py - crm/test_crm_available.py + - decap/test_decap.py - dns/static_dns/test_static_dns.py - dns/test_dns_resolv_conf.py - drop_packets/test_drop_counters.py @@ -691,6 +692,7 @@ t1-lag-vpp: - ipfwd/test_mtu.py - ipfwd/test_dip_sip.py - lldp/test_lldp.py + - lldp/test_lldp_syncd.py - log_fidelity/test_bgp_shutdown.py - pc/test_po_voq.py - pc/test_lag_member.py diff --git a/.azure-pipelines/pr_test_template.yml b/.azure-pipelines/pr_test_template.yml index db38d522ea5..062cbebd8ab 100644 --- a/.azure-pipelines/pr_test_template.yml +++ b/.azure-pipelines/pr_test_template.yml @@ -33,6 +33,10 @@ parameters: type: string default: "" +- name: PTF_IMAGE_TAG + type: string + default: "" + - name: PTF_MODIFIED type: string default: "False" @@ -63,13 +67,21 @@ parameters: type: object default: {} +- name: MGMT_BRANCH + type: string + default: $(BUILD_BRANCH) + +- name: KVM_IMAGE_BRANCH + type: string + default: $(BUILD_BRANCH) + - name: KVM_BUILD_ID type: string default: "" - name: INCLUDE_JOBS type: string - default: "t0_job,t1_job,t2_job,t0_2vlans_job,t0_sonic_job,dpu_job,t1_multi_asic_job,t1_lag_vpp_job" + default: "all" jobs: - job: get_impacted_area @@ -127,15 +139,16 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t0 SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -170,6 +183,7 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t0 @@ -177,9 +191,9 @@ jobs: MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -214,15 +228,16 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t1-lag SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -257,6 +272,7 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: dualtor @@ -264,9 +280,9 @@ jobs: MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) COMMON_EXTRA_PARAMS: "--disable_loganalyzer --disable_sai_validation " - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -300,17 +316,18 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t0-64-32 SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} COMMON_EXTRA_PARAMS: "--neighbor_type=sonic --disable_sai_validation " VM_TYPE: vsonic - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} SPECIFIC_PARAM: '[ {"name": "bgp/test_bgp_fact.py", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"}, {"name": "macsec", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"} @@ -348,15 +365,16 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: dpu SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " SPECIFIC_PARAM: '[ {"name": "dash/test_dash_vnet.py", "param": "--skip_dataplane_checking"} @@ -395,6 +413,7 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t1-8-lag @@ -402,9 +421,9 @@ jobs: MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) NUM_ASIC: 4 - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -440,15 +459,16 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t2 SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} COMMON_EXTRA_PARAMS: "--disable_sai_validation " ${{ each param in parameters.OVERRIDE_PARAMS }}: ${{ param.key }}: ${{ param.value }} @@ -483,15 +503,16 @@ jobs: TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} MGMT_COMMIT_HASH: ${{ parameters.MGMT_COMMIT_HASH }} + PTF_IMAGE_TAG: ${{ parameters.PTF_IMAGE_TAG }} PTF_MODIFIED: ${{ parameters.PTF_MODIFIED }} EXPECTED_RESULT: ${{ parameters.EXPECTED_RESULT }} TOPOLOGY: t1-lag-vpp SCRIPTS: $(SCRIPTS) MIN_WORKER: $(INSTANCE_NUMBER) MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + KVM_IMAGE_BRANCH: ${{ parameters.KVM_IMAGE_BRANCH }} KVM_BUILD_ID: ${{ parameters.KVM_BUILD_ID }} - MGMT_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: ${{ parameters.MGMT_BRANCH }} ASIC_TYPE: "vpp" KVM_IMAGE_BUILD_PIPELINE_ID: "2818" COMMON_EXTRA_PARAMS: "--disable_sai_validation --disable_loganalyzer" diff --git a/.github/.code-reviewers/auto-assign.py b/.github/.code-reviewers/auto-assign.py index 3fe46a72af6..c1e79de5ae0 100644 --- a/.github/.code-reviewers/auto-assign.py +++ b/.github/.code-reviewers/auto-assign.py @@ -1,6 +1,7 @@ from collections import Counter, deque +import fnmatch import os -from shutil import unregister_unpack_format +import re import yaml from github import Auth, Github @@ -14,6 +15,95 @@ INCLUDE_CONTRIBUTORS_TIES = os.environ.get( "INCLUDE_CONTRIBUTORS_TIES", "False" ).strip().lower() not in ("", "false", "f", "0", "no", "n", "off", "disabled") +SKIP_EXPIRY_CONFIG_PATH = os.environ.get( + "SKIP_EXPIRY_CONFIG_PATH", ".github/SKIP_EXPIRY_CONFIG.yaml" +) + +CONDITIONAL_MARK_GLOB_PATTERNS = ( + "tests/common/plugins/conditional_mark/tests_mark_conditions*.yaml", + "tests/common/plugins/conditional_mark/tests_mark_conditions*.yml", +) +RELEASE_BRANCH_PATTERN = re.compile(r"^202\d{3}$") +CONDITIONAL_MARK_NOTE_MARKER = "" + + +def is_master_or_release_branch(branch_name: str) -> bool: + return branch_name in {"master", "main"} or bool( + RELEASE_BRANCH_PATTERN.fullmatch(branch_name) + ) + + +def load_skip_expiry_maintainers(config_path: str) -> list[str]: + with open(config_path, "r", encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) or {} + + maintainers = config.get("maintainers", []) + if not isinstance(maintainers, list): + return [] + + return [ + str(maintainer).strip().lstrip("@") + for maintainer in maintainers + if str(maintainer).strip() + ] + + +def matches_conditional_mark_file(filename: str) -> bool: + return any( + fnmatch.fnmatch(filename, file_pattern) + for file_pattern in CONDITIONAL_MARK_GLOB_PATTERNS + ) + + +def tag_skip_maintainers_if_change_to_conditional_mark( + pull_request, changed_files: list[str] +) -> None: + if not is_master_or_release_branch(pull_request.base.ref): + base_branch = pull_request.base.ref + print( + f"Skipping conditional-mark maintainer note: base branch '{base_branch}' " + "is not master/main/release" + ) + return + + has_conditional_mark_changes = any( + matches_conditional_mark_file(changed_file) for changed_file in changed_files + ) + if not has_conditional_mark_changes: + print( + "Skipping conditional-mark maintainer note: " + "no conditional-mark files were changed" + ) + return + + try: + maintainers = load_skip_expiry_maintainers(SKIP_EXPIRY_CONFIG_PATH) + except Exception as error: + print(f"Failed to load skip-expiry maintainers from '{SKIP_EXPIRY_CONFIG_PATH}': {error}") + return + + if not maintainers: + print("Skipping conditional-mark maintainer note: no maintainers found") + return + + existing_comments = pull_request.get_issue_comments() + for comment in existing_comments: + if CONDITIONAL_MARK_NOTE_MARKER in comment.body: + print("Conditional-mark maintainer note already exists, skipping new comment") + return + + maintainers_mentions = " ".join(f"@{maintainer}" for maintainer in maintainers) + note = ( + f"{CONDITIONAL_MARK_NOTE_MARKER}\n" + f"{maintainers_mentions} A user wants to merge changes to " + f"the conditional mark files into `{pull_request.base.ref}`. " + "Please review." + ) + pull_request.create_issue_comment(note) + print( + f"Posted conditional-mark maintainer note for PR #{pull_request.number} to: {maintainers}" + ) + # using an access token auth = Auth.Token(GITHUB_TOKEN) @@ -38,11 +128,12 @@ # Until the sufficient number of reviewers are found updated_folders = [] reviewer_candidates = Counter[str, int]() +changed_files = [changed_file.filename for changed_file in pr.get_files()] # First bring each changed path to where any reviwer exists -for changed_file in pr.get_files(): +for changed_file in changed_files: # remove the filename, add "/" to the front - changed_path = os.path.join(os.sep, os.path.dirname(changed_file.filename)) + changed_path = os.path.join(os.sep, os.path.dirname(changed_file)) print(f"Processing changed path {changed_path}") while changed_path not in reviewer_index: if changed_path in seen_folders: @@ -114,3 +205,5 @@ else: print("No reviewers found for this PR!") + +tag_skip_maintainers_if_change_to_conditional_mark(pr, changed_files) diff --git a/.github/workflows/assignReviewers.yaml b/.github/workflows/assignReviewers.yaml index e553d09b5d8..981d0afed7c 100644 --- a/.github/workflows/assignReviewers.yaml +++ b/.github/workflows/assignReviewers.yaml @@ -26,9 +26,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.pull_request.number }} REVIEWER_INDEX: .github/.code-reviewers/pr_reviewer-by-files.yml + SKIP_EXPIRY_CONFIG_PATH: .github/SKIP_EXPIRY_CONFIG.yaml NEEDED_REVIEWER_COUNT: 3 INCLUDE_CONTRIBUTORS_TIES: True - + - name: Cleanup the checked out repo run: git clean -fdx - diff --git a/ansible/TestbedProcessing.py b/ansible/TestbedProcessing.py index 736d1789fa3..b1a833f84d5 100755 --- a/ansible/TestbedProcessing.py +++ b/ansible/TestbedProcessing.py @@ -167,7 +167,7 @@ def makeMain(data, outfile): } } with open(outfile, "w") as toWrite: - toWrite.write("supported_vm_types: [ 'veos', 'ceos', 'vsonic' ]\n"), + toWrite.write("supported_vm_types: [ 'veos', 'ceos', 'vsonic', 'csonic' ]\n"), yaml.dump(dictData, stream=toWrite, default_flow_style=False) toWrite.write("# proxy\n") yaml.dump(proxy, stream=toWrite, default_flow_style=False) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index b83e0452068..ceea793ab90 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -26,7 +26,9 @@ transport = smart module_lang = C max_diff_size = 512000 -interpreter_python = auto_legacy_silent +# auto_legacy_silent is deprecated; use auto instead +# https://docs.ansible.com/projects/ansible/latest/reference_appendices/interpreter_discovery.html +interpreter_python = auto # plays will gather facts by default, which contain information about # the remote system. diff --git a/ansible/ceos_topo_converger.py b/ansible/ceos_topo_converger.py index a49078b3bb5..1b75f60df8f 100644 --- a/ansible/ceos_topo_converger.py +++ b/ansible/ceos_topo_converger.py @@ -12,6 +12,7 @@ CEOSLAB_INTF_LIMIT = 127 # 128, minus one for backplane interface BASE_VLAN_ID = 2000 +DEFAULT_MAX_FP_NUM = 4 class ListIndentDumper(yaml.Dumper): @@ -147,6 +148,12 @@ def converge_peers(self, intf_counter_base = 1 eth_intf_index = 1 offset = 0 + # Per-prime allocator for any *additional* Port-Channels (see below). + # Primary Port-Channels consume channel ids in [intf_counter_base, + # intf_counter_base + len(peer_list) - 1] (one per VRF), so start the + # extra-channel counter just above that range to keep every + # channel-group id globally unique on the prime. + next_extra_po = len(peer_list) + intf_counter_base for i, peer_name in enumerate(peer_list): # For simplicity, VRFs are just peer names. vlan_id = BASE_VLAN_ID + offset_mapping[peer_name] @@ -158,22 +165,47 @@ def converge_peers(self, intf_index = i + intf_counter_base vrf = {f"Vlan{vlan_id}": {}} + # A peer may attach to more than one Port-Channel (e.g. dualtor + # T1s peer with BOTH ToRs via two separate single-member LAGs). + # Allocate a globally-unique converged channel-group id per + # original Port-Channel: the primary (lowest-numbered) keeps + # ``intf_index`` so single-Port-Channel topologies + # (t0/t1/t2/dpu/...) render byte-identically, while any extra + # Port-Channel gets a fresh id from ``next_extra_po`` (outside the + # per-VRF primary range) so it never collides with another VRF's + # primary channel. ``lacp_remap`` maps each original channel-group + # number to its converged id so member Ethernets stay bundled with + # the correct Port-Channel. + po_names = sorted( + (name for name in peer_intfs if name.startswith("Port-Channel")), + key=lambda name: int(name[len("Port-Channel"):]), + ) + lacp_remap = {} + for po_name in po_names: + orig_ch = int(po_name[len("Port-Channel"):]) + if not lacp_remap: + lacp_remap[orig_ch] = intf_index + else: + lacp_remap[orig_ch] = next_extra_po + next_extra_po += 1 + for intf, config in peer_intfs.items(): if "Ethernet" not in intf: continue eth_intf = f"Ethernet{eth_intf_index}" vrf[eth_intf] = deepcopy(peer_intfs[intf]) # Update lacp channel-group to match the new Port-Channel index - if "lacp" in vrf[eth_intf] and "Port-Channel1" in peer_intfs: - vrf[eth_intf]["lacp"] = intf_index + if "lacp" in vrf[eth_intf] and lacp_remap: + vrf[eth_intf]["lacp"] = lacp_remap.get( + peer_intfs[intf]["lacp"], intf_index) orig_intf_map[intf] = eth_intf eth_intf_index += 1 - if "Port-Channel1" in peer_intfs: - po_intf = f"Port-Channel{intf_index}" - orig_intf_map["Port-Channel1"] = po_intf - vrf[po_intf] = deepcopy( - peer_intfs["Port-Channel1"]) + for po_name in po_names: + orig_ch = int(po_name[len("Port-Channel"):]) + po_intf = f"Port-Channel{lacp_remap[orig_ch]}" + orig_intf_map[po_name] = po_intf + vrf[po_intf] = deepcopy(peer_intfs[po_name]) if "Loopback0" in peer_intfs: lo_intf = f"Loopback{intf_index}" orig_intf_map["Loopback0"] = lo_intf @@ -226,10 +258,13 @@ def converge_topo(self) -> None: # We don't need to change the host_interfaces portion of the passed topo, so # copy - # it over as is. - key = "host_interfaces" - if key in old_topo: - new_topo[key] = old_topo[key].copy() + # it over as is. The same applies to disabled_host_interfaces, which must be + # preserved so the DUT minigraph keeps those ports admin-down; dropping it + # turns previously-disabled host interfaces into active ports and breaks + # buffer/qos deployment checks (e.g. qos/test_buffer.py). + for key in ("host_interfaces", "disabled_host_interfaces"): + if key in old_topo: + new_topo[key] = old_topo[key].copy() key = "VMs" # Save off which vm had which interface index as we will need this later @@ -247,6 +282,18 @@ def converge_topo(self) -> None: if key in old_topo: new_topo[key] = old_topo[key].copy() + # Preserve top-level topology metadata that minigraph generation reads + # directly off the topology dict (ansible/library/topo_facts.py). + # dut_num in particular sizes the per-DUT interface_indexes lists: + # multi-linecard chassis topologies (t2 variants carry dut_num 3-6) + # have VM vlans with dut_index > 0, so dropping dut_num makes + # topo_facts default it to 1 and raise "IndexError: list index out of + # range" during deploy-mg. Single-DUT topos omit dut_num (defaults to + # 1), so this is a no-op for them. + for key in ("dut_num", "topo_type"): + if key in old_topo: + new_topo[key] = old_topo[key] + new_topo = self.converged_topo old_topo = self.topo key = "configuration_properties" @@ -257,6 +304,31 @@ def converge_topo(self) -> None: new_topo[key] = old_topo[key].copy() new_topo["convergence_data"] = self.converge_peers(interface_indexes, offsets) + # After convergence, each prime cEOSLab peer needs one front-panel + # interface per merged sub-peer (each connects to one DUT FP port via + # one ``br--N`` OVS bridge). The default ``max_fp_num`` of 4 + # works for stock topologies, but converged primes routinely hold many + # more (e.g. 16 on a t1-lag spine prime, 32 on a t2 T3 prime). Without + # bumping max_fp_num, ``create_bridges`` / ``ceos_network`` create only + # 4 br-VM-N bridges + 4 veth pairs into the cEOS container, and the + # subsequent ``vm_topology bind`` fails with "Too many vlans". + # + # Surface the required count as ``max_fp_num_provided`` at the topo + # root so the vm_set role bumps max_fp_num for the whole vm_set. The + # override is applied in ``roles/vm_set/tasks/main.yml`` (common to + # every action, including ``add_topo`` which actually creates the + # br-VM-N bridges) and ``roles/vm_set/tasks/start.yml``. Cap at + # CEOSLAB_INTF_LIMIT to stay within cEOSLab's per-container interface + # ceiling, and never go below the default so single-vlan converged + # topologies (e.g. dpu) keep the existing minimum. + max_prime_vlans = max( + (len(vm.get("vlans", [])) for vm in new_topo["topology"]["VMs"].values()), + default=0, + ) + required_fp_num = max(DEFAULT_MAX_FP_NUM, min(max_prime_vlans, CEOSLAB_INTF_LIMIT)) + if required_fp_num > DEFAULT_MAX_FP_NUM: + self.converged_topo["max_fp_num_provided"] = required_fp_num + def run(self) -> None: self.parse_properties() self.converge_topo() diff --git a/ansible/config_sonic_basedon_testbed.yml b/ansible/config_sonic_basedon_testbed.yml index 8c31b06fc0f..dd08691747d 100644 --- a/ansible/config_sonic_basedon_testbed.yml +++ b/ansible/config_sonic_basedon_testbed.yml @@ -154,6 +154,10 @@ is_bmc_topo: "{{ 'bmc' in topo }}" tags: always + - name: set prober_type + set_fact: + prober_type: "{{ testbed_facts['prober_type'] | default(None) }}" + - name: set lit mode set_fact: is_lit_mode: "{{ testbed_facts.get('is_lit_mode', topo in ['t1-smartswitch-ha', 't1-28-lag', 'smartswitch-t1', 't1-48-lag']) }}" @@ -916,6 +920,7 @@ port_override_from_links: "{{ port_override_from_links | default(false) | bool }}" hwsku: "{{ (lab_csv_result.stdout | from_json).hwsku if (port_override_from_links | default(false) | bool) and lab_csv_result is defined and lab_csv_result.rc == 0 and (lab_csv_result.stdout | from_json).hwsku else hwsku }}" vm_configuration: "{{ configuration if topo == 't1-filterleaf-lag' else omit }}" + prober_type: "{{ prober_type | default(omit) }}" is_lit_mode: "{{ is_lit_mode | default(true) }}" npu_index: "{{ dut_index | default(0) | int }}" bgp_confd_asn: "{{ bgp_confd_asn }}" @@ -1158,6 +1163,11 @@ - name: Force rapid NTP polling with chrony become: true command: chronyc burst 4/4 + register: chrony_burst_result + until: chrony_burst_result.rc == 0 + retries: 3 + delay: 2 + changed_when: false - name: Wait for chrony synchronization (remaining correction < 0.5s) become: true @@ -1342,10 +1352,10 @@ when: "'dualtor-mixed' in topo or 'dualtor-aa' in topo" - - name: execute "TSB" on T2 DUTs + - name: execute "TSB" on T2/RH DUTs become: True shell: "TSB" - when: "'t2' in topo" + when: "'t2' in topo or 'rh' in topo" - name: execute cli "config save -y" to save current minigraph as startup-config become: true diff --git a/ansible/generate_topo.py b/ansible/generate_topo.py index 133d3a39c1b..a19881092ef 100755 --- a/ansible/generate_topo.py +++ b/ansible/generate_topo.py @@ -222,17 +222,19 @@ def __contains__(self, key): "skip_ports": [p for p in range(128) if p % 8 not in (0, 6)], "panel_port_step": 1}, # t0-isolated 32 DL / 32 UL / 2 peer "mix" layout. - # 16 interleaved super-blocks of 20 panel ports (4 sets of 5). Within each super-block - # keep sets 1 and 4 (offsets 0..4 and 15..19) and skip sets 2 and 3 (offsets 5..14). - # Within every kept set of 5: 1st port (offset 0/15) = downlink (host interface on t0), - # 5th port (offset 4/19) = uplink (T1 VM). 16 * 2 = 32 DL, 16 * 2 = 32 UL. - # 2 peer (pt0) VMs at panel ports 320, 321 (appended after the 320-port main range). - # NOTE: the range(320) literals are tied to the intended `-c 322` invocation - # (320 main panel ports + 2 peer ports); update them if `-c` changes. + # 16 interleaved super-blocks of 12 panel ports (4 groups of 3). Within each super-block + # keep groups 1 and 4 (offsets 0..2 and 9..11) and skip groups 2 and 3 (offsets 3..8). + # Within every kept group of 3: 1st port (offset 0/9) = downlink (host interface on t0), + # 3rd port (offset 2/11) = uplink (T1 VM); middle port (offset 1/10) is skipped. + # 16 * 2 = 32 DL, 16 * 2 = 32 UL. + # 2 peer (pt0) VMs at panel ports 192, 193 (appended after the 192-port main range). + # NOTE: the range(192) literals are tied to the intended `-c 192` invocation + # (peer_ports are appended after panel_port_count, not included in it); + # update them if `-c` changes. 'd32u32s2-mix': {"ds_breakout": 1, "us_breakout": 1, "ds_link_step": 1, "us_link_step": 1, - "uplink_ports": [p for p in range(320) if p % 20 in (4, 19)], - "peer_ports": [320, 321], - "skip_ports": [p for p in range(320) if p % 20 not in (0, 4, 15, 19)], + "uplink_ports": [p for p in range(192) if p % 12 in (2, 11)], + "peer_ports": [192, 193], + "skip_ports": [p for p in range(192) if p % 12 not in (0, 2, 9, 11)], "panel_port_step": 1}, # t1-isolated with 28 downlinks + 4 uplinks (no peers) on a 32-port device. # Panel ports 0..27 = T0 downlinks; panel ports 28..31 = T2 uplinks. @@ -778,7 +780,7 @@ def main(role: str, keyword: str, template: str, port_count: int, uplinks: str, - ./generate_topo.py -r t1 -k isolated -t t1-isolated -c 509 -l 'd508u1s2' - ./generate_topo.py -r t1 -k isolated -t t1-isolated -c 509 -l 'd32u1s2' # 509 matches d508u1s2 IPs - ./generate_topo.py -r t1 -k isolated -t t1-isolated -c 128 -l 'd32' # 32 DL only - - ./generate_topo.py -r t0 -k isolated-mix -t t0-isolated -c 320 -l 'd32u32s2-mix' # mix layout + - ./generate_topo.py -r t0 -k isolated-mix -t t0-isolated -c 192 -l 'd32u32s2-mix' # mix layout - ./generate_topo.py -r t1 -k isolated -t t1-isolated -c 32 -l 'd28u4' # 28 DL + 4 UL - ./generate_topo.py -r lt2 -k o128 -t lt2_128 -c 64 -l 'o128lt2' - ./generate_topo.py -r lt2 -k p32o64 -t lt2_p32o64 -c 64 -l 'p32o64lt2' diff --git a/ansible/group_vars/all/creds.yml b/ansible/group_vars/all/creds.yml index 5bf4a23e27e..82eabee7e68 100644 --- a/ansible/group_vars/all/creds.yml +++ b/ansible/group_vars/all/creds.yml @@ -30,4 +30,4 @@ vm_host_user: use_own_value vm_host_password: use_own_value vm_host_become_password: use_own_value -csonic_image: docker-sonic-vs +csonic_image: "{{ lookup('env', 'SONIC_MGMT_CSONIC_DOCKER_IMAGE') | default('docker-sonic-vs', true) }}" diff --git a/ansible/group_vars/ptf/secrets.yml b/ansible/group_vars/ptf/secrets.yml index 6b34c12f6ae..0eded2032f6 100644 --- a/ansible/group_vars/ptf/secrets.yml +++ b/ansible/group_vars/ptf/secrets.yml @@ -1,7 +1,8 @@ ansible_connection: multi_passwd_ssh -ansible_user: root -ansible_ssh_pass: root +ansible_user: "{{ lookup('env', 'SONIC_MGMT_PTF_USER') | default('root', true) }}" +ansible_password: "{{ lookup('env', 'SONIC_MGMT_PTF_PASSWORD') | default('root', true) }}" +ansible_ssh_pass: "{{ lookup('env', 'SONIC_MGMT_PTF_PASSWORD') | default('root', true) }}" # ansible_altpasswords: # - fakepassword1 # - fakepassword2 diff --git a/ansible/group_vars/sonic/variables b/ansible/group_vars/sonic/variables index 040d4ed87bf..a1c2fc33571 100644 --- a/ansible/group_vars/sonic/variables +++ b/ansible/group_vars/sonic/variables @@ -1,6 +1,6 @@ ansible_ssh_user: admin ansible_connection: multi_passwd_ssh -ansible_altpassword: YourPaSsWoRd +ansible_altpassword: "{{ lookup('env', 'SONIC_MGMT_SONIC_ALTPASSWORD') | default('YourPaSsWoRd', true) }}" # ansible_altpasswords: # - fakepassword1 # - fakepassword2 diff --git a/ansible/group_vars/vm_host/creds.yml b/ansible/group_vars/vm_host/creds.yml index 327da266185..faa24913003 100644 --- a/ansible/group_vars/vm_host/creds.yml +++ b/ansible/group_vars/vm_host/creds.yml @@ -1,10 +1,10 @@ --- -ansible_user: use_own_value -ansible_password: use_own_value -ansible_become_password: use_own_value +ansible_user: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_USER') | default('use_own_value', true) }}" +ansible_password: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_PASSWORD') | default('use_own_value', true) }}" +ansible_become_password: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_BECOME_PASSWORD') | default('use_own_value', true) }}" # Use the following username/password variables to login to vm hosts # instead of the default variables (defined above). -vm_host_user: use_own_value -vm_host_password: use_own_value -vm_host_become_password: use_own_value +vm_host_user: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_USER') | default('use_own_value', true) }}" +vm_host_password: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_PASSWORD') | default('use_own_value', true) }}" +vm_host_become_password: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_BECOME_PASSWORD') | default('use_own_value', true) }}" diff --git a/ansible/group_vars/vm_host/csonic.yml b/ansible/group_vars/vm_host/csonic.yml index 84227a1104b..d8b6c6bc6e0 100644 --- a/ansible/group_vars/vm_host/csonic.yml +++ b/ansible/group_vars/vm_host/csonic.yml @@ -1,2 +1,10 @@ -csonic_image: docker-sonic-vs +csonic_image: "{{ lookup('env', 'SONIC_MGMT_CSONIC_DOCKER_IMAGE') | default('docker-sonic-vs', true) }}" csonic_image_pull: false + +# Net base container holds the network namespace that the cSONiC container +# joins (network_mode: container:net_...). It only needs to stay alive and +# carry net_admin; any maintained, small base image works. +# debian:jessie was EOL in 2019 and is no longer pullable, so default to a +# supported slim image. pull=false means pull only when absent locally. +csonic_netbase_image: debian:bookworm-slim +csonic_netbase_image_pull: false diff --git a/ansible/library/announce_routes.py b/ansible/library/announce_routes.py index 72a7277c03b..bbc728b41ca 100644 --- a/ansible/library/announce_routes.py +++ b/ansible/library/announce_routes.py @@ -743,7 +743,7 @@ def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce if aggregate_routes_v4: filterout_subnet_ipv4(aggregate_routes, routes_v4) routes_v4.extend(aggregate_routes_v4) - topo_routes[k][IPV4] = routes_v4 + topo_routes[k][IPV4] = topo_routes[k].get(IPV4, []) + routes_v4 routes_to_change[port] += routes_v4 if enable_ipv6_routes_generation: routes_v6, _ = generate_routes("v6", podset_number, tor_number, tor_subnet_number, @@ -756,7 +756,7 @@ def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce if aggregate_routes_v6: filterout_subnet_ipv6(aggregate_routes, routes_v6) routes_v6.extend(aggregate_routes_v6) - topo_routes[k][IPV6] = routes_v6 + topo_routes[k][IPV6] = topo_routes[k].get(IPV6, []) + routes_v6 routes_to_change[port6] += routes_v6 if 'vips' in v: @@ -1566,17 +1566,20 @@ def fib_lt2_routes(topo, ptf_ip, action="annouce", topo_routes=None): group_nums = int(math.ceil(float(len(t1_vms)) / T1_GROUP_SIZE)) t1_route_per_group = int(math.ceil(ROUTE_NUMBER_T1 / T1_GROUP_SIZE / group_nums)) - # 32 routes each x 8 to support up to 256 T1 VMs - extra_ipv4_t1 = itertools.chain( - ipaddress.ip_network("192.168.0.0/27"), - ipaddress.ip_network("192.169.0.0/27"), - ipaddress.ip_network("192.170.0.0/27"), - ipaddress.ip_network("192.171.0.0/27"), - ipaddress.ip_network("192.172.0.0/27"), - ipaddress.ip_network("192.173.0.0/27"), - ipaddress.ip_network("192.174.0.0/27"), - ipaddress.ip_network("192.175.0.0/27"), + # 32 addresses per /27; need ceil(len(t1_vms)/32) blocks (min 4). + # 224 T1 (needs 7); build enough 192.(168+i).0.0/27 nets. + num_extra = max(4, int(math.ceil(float(len(t1_vms)) / 32))) + # Second octet 168+i must stay <= 255, so i <= 87 and num_extra <= 88 (~2816 T1 VMs). + assert num_extra <= 88, ( + "fib_lt2_routes: num_extra={} ({} T1 VMs) exceeds scheme limit 88 for 192.(168+i).0.0/27" + .format(num_extra, len(t1_vms)) ) + extra_networks = [] + for i in range(num_extra): + extra_networks.append( + ipaddress.ip_network(UNICODE_TYPE("192.{}.0.0/27".format(168 + i))) + ) + extra_ipv4_t1 = itertools.chain(*extra_networks) for group in range(group_nums): selected_v4_subnets = all_subnetv4[group * t1_route_per_group: group * t1_route_per_group + t1_route_per_group] diff --git a/ansible/library/bgp_facts.py b/ansible/library/bgp_facts.py index a22423414e9..ddf946f9772 100644 --- a/ansible/library/bgp_facts.py +++ b/ansible/library/bgp_facts.py @@ -1,6 +1,7 @@ #!/usr/bin/python from ansible.module_utils.basic import AnsibleModule import re +import time DOCUMENTATION = ''' module: bgp_facts @@ -98,19 +99,36 @@ def collect_data(self, command_str, instance): """ Collect bgp information by reading output of 'vtysh' command line tool """ - docker_cmd = 'docker exec -i {} vtysh -c "show ip bgp {}" '.format( - instance, command_str) - try: - rc, self.out, err = self.module.run_command( - docker_cmd, executable='/bin/bash', use_unsafe_shell=True) - except Exception as e: - self.module.fail_json(msg=str(e)) - - if rc != 0: - self.module.fail_json(msg="Command failed rc=%d, out=%s, err=%s" % - (rc, self.out, err)) - - return + deadline = time.time() + 120 + interval = 2 + last_rc, last_err = None, '' + while True: + remaining = deadline - time.time() + if remaining <= 0: + break + per_call = min(20, max(5, int(remaining))) + docker_cmd = 'timeout -k 5 {t} docker exec -i {inst} vtysh -c "show ip bgp {cmd}" '.format( + t=per_call, inst=instance, cmd=command_str) + try: + rc, self.out, err = self.module.run_command( + docker_cmd, executable='/bin/bash', use_unsafe_shell=True) + except Exception as e: + self.module.fail_json(msg=str(e)) + + if rc == 0: + return + last_rc, last_err = rc, err + + sleep_for = min(interval, max(0, deadline - time.time())) + if sleep_for <= 0: + break + time.sleep(sleep_for) + interval = min(interval * 2, 10) + + self.module.fail_json( + msg="bgp_facts: 'show ip bgp %s' in container '%s' did not succeed within 120s " + "(last rc=%s, err=%r); caller should retry" % ( + command_str, instance, last_rc, last_err)) def parse_summary(self): regex_asn = re.compile(r'.*local AS number (\d+).*') diff --git a/ansible/library/extract_log.py b/ansible/library/extract_log.py index 2594ecd2e76..b1633803419 100644 --- a/ansible/library/extract_log.py +++ b/ansible/library/extract_log.py @@ -169,7 +169,7 @@ def convert_date(fct, s): locale.setlocale(locale.LC_ALL, loc) if dt is None: - dt = 0 + dt = datetime.datetime.min logger.warning(f"Failed to convert date from string, skipping unparseable line: {s}") return dt diff --git a/ansible/library/generate_golden_config_db.py b/ansible/library/generate_golden_config_db.py index 721aa414ef6..3667347c070 100644 --- a/ansible/library/generate_golden_config_db.py +++ b/ansible/library/generate_golden_config_db.py @@ -182,6 +182,7 @@ def __init__(self): num_asics=dict(required=False, type='int', default=1), hwsku=dict(required=False, type='str', default=None), vm_configuration=dict(required=False, type='dict', default={}), + prober_type=dict(required=False, type='str', default=None), is_lit_mode=dict(required=False, type='bool', default=True), npu_index=dict(required=False, type='int', default=0), duts_list=dict(required=False, type='list', default=[]), @@ -205,6 +206,7 @@ def __init__(self): self.hwsku = dut_hwsku self.vm_configuration = self.module.params['vm_configuration'] + self.prober_type = self.module.params['prober_type'] self.is_lit_mode = self.module.params['is_lit_mode'] self.bgp_confd_asn = self.module.params['bgp_confd_asn'] self.bgp_confd_peers = self.module.params['bgp_confd_peers'] @@ -403,12 +405,17 @@ def get_multiasic_feature_config(self): def overwrite_feature_golden_config_db_multiasic(self, config, feature_key, auto_restart="enabled", state="enabled", feature_data=None): full_config = json.loads(config) - if full_config == {} or "FEATURE" not in full_config.get("localhost", {}): - # need dump running config FEATURE + selected feature - gold_config_db = self.get_multiasic_feature_config() - else: - # need existing config + selected feature - gold_config_db = full_config + if "FEATURE" not in full_config.get("localhost", {}): + # Merge running config FEATURE into existing config instead of replacing, + # to preserve other tables (e.g. BGP_DEVICE_GLOBAL) already in full_config. + feature_config = self.get_multiasic_feature_config() + for ns, ns_data in feature_config.items(): + if ns in full_config: + full_config[ns].update(ns_data) + else: + full_config[ns] = ns_data + + gold_config_db = full_config if feature_data is None: feature_data = { @@ -908,7 +915,8 @@ def _generate_ha_config(self, dpu_num, enabled_dpu_set): "Vnet_55": { "scope": "default", "vni": "10000", - "vxlan_tunnel": "t4" + "vxlan_tunnel": "t4", + "advertise_prefix": "true" } }, "VXLAN_TUNNEL": { @@ -1116,6 +1124,36 @@ def generate_lt2_ft2_golden_config_db(self): return json.dumps(golden_config, indent=4) + def generate_dualtor_golden_config_db(self): + """ + Generate golden config for dualtor topology with prober_type support. + This adds prober_type to existing MUX_CABLE entries from minigraph. + """ + rc, out, err = self.module.run_command("sonic-cfggen -H -m -j /etc/sonic/init_cfg.json --print-data") + if rc != 0: + self.module.fail_json(msg="Failed to get config from minigraph: {}".format(err)) + # Get existing config from minigraph + ori_config_db = json.loads(out) + golden_config_db = {} + + # Preserve DEVICE_METADATA + if "DEVICE_METADATA" in ori_config_db: + golden_config_db["DEVICE_METADATA"] = ori_config_db["DEVICE_METADATA"] + golden_config_db["DEVICE_METADATA"]["localhost"]["buffer_model"] = "traditional" + + # Add prober_type to MUX_CABLE if it exists and prober_type is specified + if ("MUX_CABLE" in ori_config_db and "PORT" in ori_config_db + and self.prober_type != "" and self.prober_type is not None): + mux_cable_config = copy.deepcopy(ori_config_db["MUX_CABLE"]) + port_config = copy.deepcopy(ori_config_db["PORT"]) + # Add prober_type to each interface + for intf_name, intf_config in mux_cable_config.items(): + intf_config["prober_type"] = self.prober_type + golden_config_db["MUX_CABLE"] = mux_cable_config + golden_config_db["PORT"] = port_config + + return json.dumps(golden_config_db, indent=4) + def override_port_table_from_platform(self, config): """ Rebuild the PORT table from port_speeds + platform.json. @@ -1225,6 +1263,9 @@ def generate(self): config = self.generate_full_lossy_golden_config_db() elif self.topo_name in ["t1-filterleaf-lag"]: config = self.generate_filterleaf_golden_config_db() + elif "dualtor" in self.topo_name: + config = self.generate_dualtor_golden_config_db() + module_msg = module_msg + " for dualtor" elif "c0" in self.topo_name: config = self.generate_c0_golden_config_db() module_msg = module_msg + " for c0" diff --git a/ansible/library/ocs_cross_connect.py b/ansible/library/ocs_cross_connect.py new file mode 100644 index 00000000000..cc1bf476723 --- /dev/null +++ b/ansible/library/ocs_cross_connect.py @@ -0,0 +1,335 @@ +#!/usr/bin/python +# Configure OCS cross-connects on an L1 (FanoutL1Sonic) switch using the OCS CLI, +# instead of building a JSON patch and reloading the config. +# +# All of the parsing and set math is implemented in Python here so the playbook task is +# a single, declarative module invocation. +# +# Algorithm (mirrors templates/config_patch/l1/ocs_xconnect.json.j2 for the desired set): +# 1. Read the current configured cross-connects ("show ocs cross-connect config") +# and the live hardware status ("show ocs cross-connect status"). +# 2. Compute the desired cross-connects from the cross_connects mapping. Each +# (a, b) entry maps to two directed cross-connects: {a}A-{b}B and {b}A-{a}B. +# 3. Clear "stale" cross-connects that exist in hardware status but not in config +# and that block a desired port. A status-only entry cannot be deleted directly, +# so it is cleared by an add-then-delete sequence. +# 4. Remove current cross-connects that conflict (occupy an A or B side a still +# missing desired cross-connect needs) so the new ones can be added. +# 5. Add the desired cross-connects that are not already present. +# 6. Optionally verify the operational status ("show ocs cross-connect status") +# reports "tuned" on both sides for every desired cross-connect. +# +# The operation is idempotent: when every desired A-B connection already exists the +# add/remove/clear lists are all empty and nothing is changed. + +import re +import time + +from ansible.module_utils.basic import AnsibleModule + +DOCUMENTATION = r''' +--- +module: ocs_cross_connect +version_added: "1.0" +short_description: Configure OCS cross-connects on an L1 (FanoutL1Sonic) switch via live CLI. +description: + - Configure the OCS cross-connects on an L1 switch to match a desired mapping using + the "config ocs cross-connect" CLI, instead of building a JSON patch and reloading. + - The OCS CLI persists changes automatically, so no "config save" is required. + - The operation is idempotent. When every desired cross-connect already exists nothing + is changed. +options: + cross_connects: + description: + - Mapping of bare A-side port number to bare B-side port number for this device, + e.g. device_l1_cross_connects[inventory_hostname]. + - Each (a, b) entry expands to two directed cross-connects {a}A-{b}B and {b}A-{a}B. + required: True + type: dict + clear_stale: + description: Clear status-only entries that block a desired port via add-then-delete. + required: False + type: bool + default: True + verify: + description: Verify status is "tuned" on both sides for every desired cross-connect. + required: False + type: bool + default: True + verify_retries: + description: Number of times to poll the status table while verifying. + required: False + type: int + default: 12 + verify_delay: + description: Seconds to wait between status polls while verifying. + required: False + type: int + default: 5 +notes: + - Run with become so the "config ocs cross-connect" commands have the required privileges. + - Supports check mode, in which the planned changes are computed and returned but not applied. +''' + +EXAMPLES = r''' +- name: deploy L1 OCS cross-connects via live CLI + become: true + ocs_cross_connect: + cross_connects: "{{ device_l1_cross_connects[inventory_hostname] | default({}) }}" + clear_stale: true + verify: true + register: ocs_reconcile +''' + +RETURN = r''' +added: + description: Ids of cross-connects that were (or would be, in check mode) added. + returned: always + type: list +removed: + description: Ids of conflicting cross-connects that were (or would be) removed. + returned: always + type: list +cleared_stale: + description: Ids of stale status-only cross-connects that were (or would be) cleared. + returned: always + type: list +desired: + description: Ids of all desired cross-connects. + returned: always + type: list +''' + +# A cross-connect id looks like "A-B", e.g. "29A-1B". +XCONN_ID_RE = re.compile(r'^[0-9]+[AB]-[0-9]+[AB]$') +NUMERIC_PORT_RE = re.compile(r'^[0-9]+$') +SHOW_CONFIG_CMD = 'show ocs cross-connect config' +SHOW_STATUS_CMD = 'show ocs cross-connect status' +ADD_CMD_TMPL = 'config ocs cross-connect add {id} update' +DEL_CMD_TMPL = 'config ocs cross-connect delete {id}' + + +def build_desired(cross_connects): + """Expand the bare a->b mapping into the list of directed cross-connects. + + Each (a, b) entry yields {a}A-{b}B and {b}A-{a}B. + """ + desired = [] + for a, b in cross_connects.items(): + a = str(a) + b = str(b) + desired.append({'id': '{}A-{}B'.format(a, b), 'a_side': '{}A'.format(a), 'b_side': '{}B'.format(b)}) + desired.append({'id': '{}A-{}B'.format(b, a), 'a_side': '{}A'.format(b), 'b_side': '{}B'.format(a)}) + return desired + + +def get_invalid_desired_ids(desired): + """Return derived desired IDs that do not match the expected xconn format.""" + return [x['id'] for x in desired if not XCONN_ID_RE.match(x['id'])] + + +def get_invalid_cross_connect_pairs(cross_connects): + """Return (a, b) entries whose ports are not bare numeric values.""" + invalid = [] + for a, b in cross_connects.items(): + sa = str(a) + sb = str(b) + if not NUMERIC_PORT_RE.match(sa) or not NUMERIC_PORT_RE.match(sb): + invalid.append({'a': sa, 'b': sb}) + return invalid + + +def parse_config(stdout): + """Parse "show ocs cross-connect config" output. + + Table format: "id a_side b_side". Skip the header ("a_side") and separator ("---") rows. + """ + xconns = [] + for line in stdout.splitlines(): + if not line.strip(): + continue + if 'a_side' in line or '---' in line: + continue + cols = line.split() + if len(cols) < 3 or not XCONN_ID_RE.match(cols[0]): + continue + xconns.append({'id': cols[0], 'a_side': cols[1], 'b_side': cols[2]}) + return xconns + + +def parse_status_ids(stdout): + """Parse the cross-connect ids (first column) from "show ocs cross-connect status". + + Status table format: "id a_side b_side a_side_status b_side_status". + """ + ids = [] + for line in stdout.splitlines(): + if not line.strip(): + continue + if 'a_side' in line or '---' in line: + continue + cols = line.split() + if not cols or not XCONN_ID_RE.match(cols[0]): + continue + ids.append(cols[0]) + return ids + + +def parse_tuned_ids(stdout): + """Return the ids whose status row reports "tuned" on both sides.""" + tuned = [] + for line in stdout.splitlines(): + norm = ' '.join(line.split()) + if norm.endswith('tuned tuned'): + tuned.append(norm.split(' ', 1)[0]) + return tuned + + +def compute_changes(desired, current, status_ids): + """Compute the stale-to-clear, conflicts-to-remove and to-add sets. + + Returns (stale_to_clear, to_remove, to_add). + """ + def _port_num(side): + return side[:-1] if side and side[-1] in ('A', 'B') else side + + desired_ids = [x['id'] for x in desired] + current_ids = [x['id'] for x in current] + desired_port_nums = set([_port_num(x['a_side']) for x in desired] + [_port_num(x['b_side']) for x in desired]) + + # Add = desired cross-connects not already present in config. Computed up front so + # that stale/conflict removals are computed independently from add operations. + to_add = [x for x in desired if x['id'] not in current_ids] + + # Stale = id present in hardware status but absent from config. Keep only those whose + # A or B port number blocks a port number a desired connect uses. + stale_to_clear = [] + for sid in status_ids: + if sid in current_ids or not XCONN_ID_RE.match(sid): + continue + parts = sid.split('-') + if _port_num(parts[0]) in desired_port_nums or _port_num(parts[1]) in desired_port_nums: + if sid not in stale_to_clear: + stale_to_clear.append(sid) + + # Conflict = a currently configured cross-connect that is not a desired pairing but + # occupies a port number (A side or B side) that a desired cross-connect uses. + to_remove = [] + seen_remove = set() + for x in current: + if x['id'] in desired_ids: + continue + if _port_num(x['a_side']) in desired_port_nums or _port_num(x['b_side']) in desired_port_nums: + if x['id'] not in seen_remove: + seen_remove.add(x['id']) + to_remove.append(x) + + return stale_to_clear, to_remove, to_add + + +def run_or_fail(module, cmd, failed_cmds): + """Run a command, recording it on non-zero return code.""" + rc, out, err = module.run_command(cmd, use_unsafe_shell=True) + if rc != 0: + failed_cmds.append({'cmd': cmd, 'rc': rc, 'stderr': err.strip()}) + return rc, out, err + + +def main(): + module = AnsibleModule( + argument_spec=dict( + cross_connects=dict(required=True, type='dict'), + clear_stale=dict(required=False, type='bool', default=True), + verify=dict(required=False, type='bool', default=True), + verify_retries=dict(required=False, type='int', default=12), + verify_delay=dict(required=False, type='int', default=5), + ), + supports_check_mode=True, + ) + p = module.params + + invalid_pairs = get_invalid_cross_connect_pairs(p['cross_connects']) + if invalid_pairs: + module.fail_json( + msg="Invalid cross_connects mapping; expected bare numeric ports for both sides.", + invalid_pairs=invalid_pairs) + + desired = build_desired(p['cross_connects']) + invalid = get_invalid_desired_ids(desired) + if invalid: + module.fail_json( + msg="Invalid cross-connect mapping; expected bare numeric ports. Invalid id(s): {}".format(invalid), + invalid=invalid) + desired_ids = [x['id'] for x in desired] + + # Read current config and live status. + rc, config_out, err = module.run_command(SHOW_CONFIG_CMD, use_unsafe_shell=True) + if rc != 0: + module.fail_json(msg="Failed to read OCS cross-connect config: {}".format(err.strip())) + rc, status_out, err = module.run_command(SHOW_STATUS_CMD, use_unsafe_shell=True) + if rc != 0: + module.fail_json(msg="Failed to read OCS cross-connect status: {}".format(err.strip())) + + current = parse_config(config_out) + status_ids = parse_status_ids(status_out) + + stale_to_clear, to_remove, to_add = compute_changes(desired, current, status_ids) + remove_ids = [x['id'] for x in to_remove] + add_ids = [x['id'] for x in to_add] + + result = dict( + desired=desired_ids, + cleared_stale=stale_to_clear, + removed=remove_ids, + added=add_ids, + ) + + changed = bool(stale_to_clear or to_remove or to_add) + + if module.check_mode: + module.exit_json(changed=changed, **result) + + add_tmpl = ADD_CMD_TMPL + del_tmpl = DEL_CMD_TMPL + failed_cmds = [] + + # A status-only ("stale") entry cannot be deleted directly; adding it to config first + # lets the subsequent delete clear it from hardware status. Run before removing config + # conflicts and adding the desired connects so the blocked ports are freed first. + if p['clear_stale']: + for xid in stale_to_clear: + run_or_fail(module, add_tmpl.replace('{id}', xid), failed_cmds) + run_or_fail(module, del_tmpl.replace('{id}', xid), failed_cmds) + + for xid in remove_ids: + run_or_fail(module, del_tmpl.replace('{id}', xid), failed_cmds) + + for xid in add_ids: + run_or_fail(module, add_tmpl.replace('{id}', xid), failed_cmds) + + if failed_cmds: + module.fail_json(msg="One or more OCS cross-connect commands failed", + failed_cmds=failed_cmds, **result) + + # Operational status: a healthy cross-connect reports "tuned" for both status columns. + # Poll until every desired cross-connect is tuned on both sides. + if p['verify'] and desired_ids: + pending = list(desired_ids) + for attempt in range(p['verify_retries']): + rc, status_out, err = module.run_command(SHOW_STATUS_CMD, use_unsafe_shell=True) + tuned = parse_tuned_ids(status_out) + pending = [d for d in desired_ids if d not in tuned] + if not pending: + break + if attempt < p['verify_retries'] - 1: + time.sleep(p['verify_delay']) + if pending: + module.fail_json( + msg="OCS cross-connects not tuned after {} retries: {}".format(p['verify_retries'], pending), + pending=pending, **result) + + module.exit_json(changed=changed, **result) + + +if __name__ == '__main__': + main() diff --git a/ansible/module_utils/debug_utils.py b/ansible/module_utils/debug_utils.py index ba5afba6dd7..28886238f68 100644 --- a/ansible/module_utils/debug_utils.py +++ b/ansible/module_utils/debug_utils.py @@ -1,7 +1,7 @@ import os import re import logging -from ansible.module_utils.basic import datetime +import datetime MAX_LOG_FILES_PER_MODULE = 10 diff --git a/ansible/plugins/connection/multi_passwd_ssh.py b/ansible/plugins/connection/multi_passwd_ssh.py index c29eeca026a..6f3e194ce6a 100644 --- a/ansible/plugins/connection/multi_passwd_ssh.py +++ b/ansible/plugins/connection/multi_passwd_ssh.py @@ -70,6 +70,13 @@ def load_source(modname, filename): # the "Permission denied" message to distinguish auth failures from # connectivity failures (timeout, no route to host, etc.). PERMISSION_DENIED_ERR_FLAG = "Permission denied" +# When a task has `no_log: true`, Ansible censors the underlying SSH stderr +# before the connection plugin sees it. In that case the error message no +# longer contains "Permission denied", so the auth-failure detection above +# would mis-classify wrong-password attempts as real connectivity failures +# and abort the retry loop. Detect the censorship marker so we can still +# iterate through remaining passwords. +NO_LOG_CENSORED_FLAG = "censored due to no log" def _password_retry(func): @@ -106,8 +113,17 @@ def _conn_with_multi_pwd(self, *args, **kwargs): # ansible-core 2.19+ with ssh_askpass raises AnsibleConnectionFailure # (not AnsibleAuthenticationFailure) for "Permission denied" auth failures. # Treat it as an auth failure so the retry loop still iterates. + # If the task sets `no_log: true`, the SSH stderr is censored before + # this plugin can inspect it, so "Permission denied" will be absent + # from `err_msg`. In that case we cannot tell auth failure from a + # real connectivity failure; assume it may be auth and keep trying + # the remaining passwords rather than aborting prematurely. err_msg = getattr(e, "message", "") or str(e) - if PERMISSION_DENIED_ERR_FLAG not in err_msg: + is_auth_failure = ( + PERMISSION_DENIED_ERR_FLAG in err_msg + or NO_LOG_CENSORED_FLAG in err_msg + ) + if not is_auth_failure: raise # not an auth failure; preserve original behaviour if not conn_passwords: raise # exhausted all passwords diff --git a/ansible/roles/eos/tasks/ceos_config.yml b/ansible/roles/eos/tasks/ceos_config.yml index de7f9689851..4a2636b89f6 100644 --- a/ansible/roles/eos/tasks/ceos_config.yml +++ b/ansible/roles/eos/tasks/ceos_config.yml @@ -57,3 +57,20 @@ template: src="{{ base_topo }}-{{ props.swrole }}.j2" dest="/{{ ceos_image_mount_dir }}/ceos_{{ vm_set_name }}_{{ inventory_hostname }}/startup-config" delegate_to: "{{ VM_host[0] }}" + +# Converged-only: append a stock-compat backplane shim (untagged VLAN 1 path, +# Vlan1 SVI in the prime peer's VRF carrying its bp_interface IP, and BGP +# network advertisement of that subnet). Keeps the existing t0-leaf.j2 / +# t1-lag-*.j2 templates untouched. Skipped on stock topologies and on +# converged hosts that have no bp_interface (e.g. servers). +- name: append converged backplane stock-compat shim to startup-config + become: yes + blockinfile: + path: "/{{ ceos_image_mount_dir }}/ceos_{{ vm_set_name }}_{{ inventory_hostname }}/startup-config" + marker: "! {mark} CONVERGED BACKPLANE STOCK-COMPAT SHIM" + block: "{{ lookup('template', 'ceos_bp_compat.j2') }}" + insertbefore: EOF + delegate_to: "{{ VM_host[0] }}" + when: + - topo_is_multi_vrf | default(false) | bool + - configuration[hostname]['bp_interface'] is defined diff --git a/ansible/roles/eos/templates/ceos_bp_compat.j2 b/ansible/roles/eos/templates/ceos_bp_compat.j2 new file mode 100644 index 00000000000..2b238fbbe99 --- /dev/null +++ b/ansible/roles/eos/templates/ceos_bp_compat.j2 @@ -0,0 +1,58 @@ +{# + Converged backplane stock-compat shim. + + Rendered ONLY for converged topologies (topo_is_multi_vrf=true) and only + for peer cEOS hosts that have a bp_interface defined. Appended to the + cEOS startup-config AFTER the main role template renders. + + Purpose: stock tests (e.g. bgp.test_bgp_stress_link_flap monitor) reach + the DUT from PTF via the legacy untagged backplane path + PTF backplane (10.10.246.254/22) -> bp_bridge -> cEOS -> DUT + In converged mode the per-VRF VLAN sub-interface model leaves untagged + ingress unresolved. This shim restores that path by: + + 1. Allowing untagged frames (VLAN 1) on the trunk backplane port. + 2. Adding an interface Vlan1 SVI in the prime peer's own VRF carrying + the original bp_interface IP (matches what stock kvm-t0 advertises). + 3. Advertising the bp_interface subnet from the same prime VRF's BGP + process so DUT learns the return path to PTF .254. + + EOS startup-config semantics MERGE successive blocks targeting the same + interface / router bgp instance, so the additions below stack onto the + earlier per-VRF config emitted by the main template without conflict. +#} +{% set host = configuration[hostname] %} +{% set conv_config = convergence_data["converged_peers"][hostname] %} +! +! ===== converged-vrf stock-compat backplane shim ===== +! +interface {{ bp_ifname }} + switchport trunk allowed vlan add 1 +! +interface Vlan1 + description {{ hostname }} legacy backplane + vrf {{ hostname }} +{% if host['bp_interface']['ipv4'] is defined %} + ip address {{ host['bp_interface']['ipv4'] }} +{% endif %} +{% if host['bp_interface']['ipv6'] is defined %} + ipv6 enable + ipv6 address {{ host['bp_interface']['ipv6'] }} + ipv6 nd ra suppress +{% endif %} + no shutdown +! +router bgp {{ conv_config['bgp']['asn'] }} + vrf {{ hostname }} + address-family ipv4 +{% if host['bp_interface']['ipv4'] is defined %} + network {{ host['bp_interface']['ipv4'] | ansible.utils.ipaddr('subnet') }} +{% endif %} + exit + address-family ipv6 +{% if host['bp_interface']['ipv6'] is defined %} + network {{ host['bp_interface']['ipv6'] | ansible.utils.ipaddr('subnet') }} +{% endif %} + exit + exit +! diff --git a/ansible/roles/eos/templates/ceos_converged.j2 b/ansible/roles/eos/templates/ceos_converged.j2 new file mode 100644 index 00000000000..c873fefda40 --- /dev/null +++ b/ansible/roles/eos/templates/ceos_converged.j2 @@ -0,0 +1,189 @@ +{# + Shared converged (multi-VRF) cEOS startup-config -- single source of truth. + + Rendered for ANY converged topology (topo_is_multi_vrf=true) regardless of + base topo / swrole, because the converged config is a pure function of the + converger output (convergence_data) plus per-peer configuration; it does not + depend on the per-topology stock template. Topologies whose per-topo template + never grew a converged branch (t0-64-32, t2, dpu, ...) include this file so + every merged sub-peer's interface IP and BGP neighbor is rendered instead of + being silently dropped. Logic mirrors the converged path already proven on + t0-leaf.j2 / t1-lag-*.j2. +#} +{% set host = configuration[hostname] %} + {% set conv_config = convergence_data["converged_peers"][hostname] %} + {% set conv_mapping = convergence_data["convergence_mapping"][hostname] %} + {% set ptf_bp_addrs = convergence_data["ptf_backplane_addrs"] %} +{% set mgmt_ip = ansible_host %} +{% if vm_type is defined and vm_type == "ceos" %} + {% set mgmt_if_index = 0 %} +{% else %} + {% set mgmt_if_index = 1 %} +{% endif %} +no schedule tech-support +! +{% if vm_type is defined and vm_type == "ceos" %} +agent LicenseManager shutdown +agent PowerFuse shutdown +agent PowerManager shutdown +agent Thermostat shutdown +agent LedPolicy shutdown +agent StandbyCpld shutdown +agent Bfd shutdown +{% endif %} +! +hostname {{ hostname }} +! +vlan {{ ptf_bp_addrs|allowed_vlans_range_str(conv_mapping) }} +! +vrf instance MGMT + {% for vrf_name in conv_config['vrf'] %} +vrf instance {{vrf_name}} + {% endfor %} +! +spanning-tree mode mstp +! +aaa root secret 0 123456 +! +username admin privilege 15 role network-admin secret 0 123456 +! +clock timezone UTC +! +lldp run +lldp management-address Management{{ mgmt_if_index }} +lldp management-address vrf MGMT +! +snmp-server community {{ snmp_rocommunity }} ro +snmp-server vrf MGMT +! +ip routing +ip routing vrf MGMT + {% for vrf_name in conv_config['vrf'] %} +ip routing vrf {{vrf_name}} + {% endfor %} +ipv6 unicast-routing + {% for vrf_name in conv_config['vrf'] %} +ipv6 unicast-routing vrf {{vrf_name}} + {% endfor %} +! +{% if disable_ceos_mgmt_gateway is defined and disable_ceos_mgmt_gateway == 'yes'%} +{% elif vm_mgmt_gw is defined %} +ip route vrf MGMT 0.0.0.0/0 {{ vm_mgmt_gw }} +{% else %} +ip route vrf MGMT 0.0.0.0/0 {{ mgmt_gw }} +{% endif %} +! +interface Management {{ mgmt_if_index }} + description TO LAB MGMT SWITCH +{% if vm_type is defined and vm_type == "ceos" %} + vrf MGMT +{% else %} + vrf forwarding MGMT +{% endif %} + ip address {{ mgmt_ip }}/{{ mgmt_prefixlen }} + no shutdown +! +interface {{ bp_ifname }} +! + description backplane + mtu 9214 + switchport trunk allowed vlan {{ ptf_bp_addrs|allowed_vlans_range_str(conv_mapping) }} + switchport mode trunk + no shutdown +! + {% for vrf in conv_mapping %} + {% for if_name, if_data in conv_config['vrf'][vrf].items() %} +interface {{ if_name }} + {% if if_name.startswith('Vlan') %} + description {{ vrf }} backplane + {% endif %} + {% if if_name.startswith('Loopback') %} + description {{ vrf }} LOOPBACK + {% endif %} + {% if if_name.startswith('Port-Channel') %} + port-channel min-links 1 + mtu 9214 + no switchport + {% endif %} + {% if if_name.startswith('Ethernet') %} + mtu 9214 + no switchport + {% endif %} + vrf {{ vrf }} + {% if if_data['lacp'] is defined %} + channel-group {{ if_data['lacp'] }} mode active + lacp rate normal + {% endif %} + {% if if_data['ipv4'] is defined %} + ip address {{ if_data['ipv4'] }} + {% endif %} + {% if if_data['ipv6'] is defined %} + ipv6 enable + ipv6 address {{ if_data['ipv6'] }} + ipv6 nd ra suppress + ipv6 nd dad disabled + {% endif %} + no shutdown +! + {% endfor %} + {% endfor %} +! +router bgp {{ conv_config['bgp']['asn'] }} + vrf MGMT + rd 1:1 + exit + {% for vrf in conv_config['vrf'] %} + vrf {{ vrf }} + {% set vrf_bgp = configuration[vrf]['bgp'] %} + {% set vrf_intfs = conv_config['vrf'][vrf] %} + {% set lo_config = vrf_intfs|get_first_loopback %} + {% if vrf_bgp['router-id'] is defined %} + router-id {{ vrf_bgp['router-id'] }} + {% else %} + {% if lo_config['ipv4'] is defined %} + router-id {{ lo_config['ipv4']|ansible.utils.ipaddr('address') }} + {% endif %} + {% endif %} + local-as {{ vrf_bgp['asn'] }} + {% for asn, remote_ips in vrf_bgp['peers'].items() %} + {% for remote_ip in remote_ips %} + neighbor {{ remote_ip }} remote-as {{ asn }} + neighbor {{ remote_ip }} description {{ asn }} + neighbor {{ remote_ip }} next-hop-self + {% if remote_ip|ipv6 %} + address-family ipv6 + neighbor {{ remote_ip }} activate + exit + {% endif %} + {% endfor %} + {% endfor %} + {% if props.enable_ipv4_routes_generation is not defined or props.enable_ipv4_routes_generation %} + neighbor {{ ptf_bp_addrs[vrf]['ipv4'] | ansible.utils.ipaddr('address') }} remote-as {{ vrf_bgp['asn'] }} + neighbor {{ ptf_bp_addrs[vrf]['ipv4'] | ansible.utils.ipaddr('address') }} next-hop-peer + neighbor {{ ptf_bp_addrs[vrf]['ipv4'] | ansible.utils.ipaddr('address') }} description exabgp_v4 + {% endif %} + {% if props.enable_ipv6_routes_generation is not defined or props.enable_ipv6_routes_generation %} + neighbor {{ ptf_bp_addrs[vrf]['ipv6'] | ansible.utils.ipaddr('address') }} remote-as {{ vrf_bgp['asn'] }} + neighbor {{ ptf_bp_addrs[vrf]['ipv6'] | ansible.utils.ipaddr('address') }} next-hop-peer + neighbor {{ ptf_bp_addrs[vrf]['ipv6'] | ansible.utils.ipaddr('address') }} description exabgp_v6 + {% endif %} + address-family ipv4 + {% if lo_config['ipv4'] is defined %} + network {{ lo_config['ipv4'] }} + {% endif %} + exit + address-family ipv6 + neighbor {{ ptf_bp_addrs[vrf]['ipv6'] | ansible.utils.ipaddr('address') }} activate + {% if lo_config['ipv6'] is defined %} + network {{ lo_config['ipv6'] }} + {% endif %} + exit + exit + {% endfor %} +! +management api http-commands + no protocol https + protocol http + no shutdown +! +end diff --git a/ansible/roles/eos/templates/dpu-tor.j2 b/ansible/roles/eos/templates/dpu-tor.j2 index 48d5d778bd4..63333a501a4 100644 --- a/ansible/roles/eos/templates/dpu-tor.j2 +++ b/ansible/roles/eos/templates/dpu-tor.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% if vm_type is defined and vm_type == "ceos" %} @@ -138,3 +143,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/eos/templates/dualtor-aa-vpp-leaf.j2 b/ansible/roles/eos/templates/dualtor-aa-vpp-leaf.j2 new file mode 120000 index 00000000000..14f30466e24 --- /dev/null +++ b/ansible/roles/eos/templates/dualtor-aa-vpp-leaf.j2 @@ -0,0 +1 @@ +dualtor-aa-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/eos/templates/dualtor-vpp-leaf.j2 b/ansible/roles/eos/templates/dualtor-vpp-leaf.j2 new file mode 120000 index 00000000000..0312b84bddb --- /dev/null +++ b/ansible/roles/eos/templates/dualtor-vpp-leaf.j2 @@ -0,0 +1 @@ +dualtor-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/eos/templates/t0-leaf-lag-2.j2 b/ansible/roles/eos/templates/t0-leaf-lag-2.j2 index 4fafef329dc..34f76106875 100644 --- a/ansible/roles/eos/templates/t0-leaf-lag-2.j2 +++ b/ansible/roles/eos/templates/t0-leaf-lag-2.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% if vm_type is defined and vm_type == "ceos" %} @@ -138,3 +143,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/eos/templates/t1-8-lag-spine.j2 b/ansible/roles/eos/templates/t1-8-lag-spine.j2 index 7022257b149..a553db54a07 100644 --- a/ansible/roles/eos/templates/t1-8-lag-spine.j2 +++ b/ansible/roles/eos/templates/t1-8-lag-spine.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% if vm_type is defined and vm_type == "ceos" %} @@ -136,3 +141,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/eos/templates/t1-8-lag-tor.j2 b/ansible/roles/eos/templates/t1-8-lag-tor.j2 index b9616aed47c..b44974f465d 100644 --- a/ansible/roles/eos/templates/t1-8-lag-tor.j2 +++ b/ansible/roles/eos/templates/t1-8-lag-tor.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% set tornum = host['tornum'] %} @@ -140,3 +145,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/eos/templates/t2-core.j2 b/ansible/roles/eos/templates/t2-core.j2 index bb4e903d00f..6a7f9a40dc9 100644 --- a/ansible/roles/eos/templates/t2-core.j2 +++ b/ansible/roles/eos/templates/t2-core.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% if vm_type is defined and vm_type == "ceos" %} @@ -160,3 +165,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/eos/templates/t2-leaf.j2 b/ansible/roles/eos/templates/t2-leaf.j2 index a41254541d6..8152a70a095 100644 --- a/ansible/roles/eos/templates/t2-leaf.j2 +++ b/ansible/roles/eos/templates/t2-leaf.j2 @@ -1,3 +1,8 @@ +{# Converged (multi-VRF) topologies render the shared converged config; the + stock body below is used unchanged on non-converged topologies. #} +{% if topo_is_multi_vrf | default(false) | bool %} +{% include 'ceos_converged.j2' %} +{% else %} {% set host = configuration[hostname] %} {% set mgmt_ip = ansible_host %} {% if vm_type is defined and vm_type == "ceos" %} @@ -152,3 +157,4 @@ management api http-commands no shutdown ! end +{% endif %} diff --git a/ansible/roles/sonic/tasks/csonic.yml b/ansible/roles/sonic/tasks/csonic.yml index c9e6957389b..86bdbb275d2 100644 --- a/ansible/roles/sonic/tasks/csonic.yml +++ b/ansible/roles/sonic/tasks/csonic.yml @@ -20,9 +20,15 @@ HWSKU: "SONiC-VM" delegate_to: "{{ VM_host[0] }}" -- name: Wait for container to be fully started - pause: - seconds: 5 +- name: Wait for CONFIG_DB to be ready in container + become: yes + command: docker exec csonic_{{ vm_set_name }}_{{ inventory_hostname }} sonic-db-cli CONFIG_DB PING + delegate_to: "{{ VM_host[0] }}" + register: configdb_ping + until: configdb_ping.rc == 0 and (configdb_ping.stdout | trim) in ["True", "PONG"] + retries: 30 + delay: 2 + changed_when: false - name: Load config_db.json into SONiC ConfigDB become: yes @@ -44,9 +50,20 @@ msg: "Config load failed: {{ config_load_result.stderr }}" when: config_load_result.rc != 0 and config_load_result.rc is defined -- name: Wait for configuration to be applied - pause: - seconds: 3 +- name: Collect front panel interfaces expected from CONFIG_DB + set_fact: + csonic_fp_interfaces: "{{ configuration[hostname]['interfaces'].keys() | select('match', '^Ethernet[0-9]+$') | list }}" + +- name: Wait for front panel interfaces to be created from CONFIG_DB + become: yes + command: docker exec csonic_{{ vm_set_name }}_{{ inventory_hostname }} ip link show {{ csonic_fp_interfaces[0] }} + delegate_to: "{{ VM_host[0] }}" + register: fp_intf_ready + until: fp_intf_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + when: csonic_fp_interfaces | length > 0 - name: Bring up front panel interface in container become: yes @@ -60,8 +77,10 @@ - name: Bring up backplane interface if defined become: yes - command: docker exec csonic_{{ vm_set_name }}_{{ inventory_hostname }} ip link set eth2 up + command: docker exec csonic_{{ vm_set_name }}_{{ inventory_hostname }} ip link set eth{{ bp_eth_idx }} up delegate_to: "{{ VM_host[0] }}" + vars: + bp_eth_idx: "{{ (configuration[hostname]['interfaces'].keys() | select('match', '^Ethernet[0-9]+$') | list | length) + 1 }}" when: configuration[hostname]['bp_interface'] is defined register: bp_ifup_result failed_when: false diff --git a/ansible/roles/sonic/tasks/csonic_config.yml b/ansible/roles/sonic/tasks/csonic_config.yml index 236558b53a6..c35aa2f346d 100644 --- a/ansible/roles/sonic/tasks/csonic_config.yml +++ b/ansible/roles/sonic/tasks/csonic_config.yml @@ -1,23 +1,3 @@ -- name: Get netbase container info - docker_container_info: - name: net_{{ vm_set_name }}_{{ inventory_hostname }} - register: ctninfo - delegate_to: "{{ VM_host[0] }}" - become: yes - -- debug: msg="{{ ctninfo.container.State.Pid }}" - -- name: Get front panel port in netbase container - shell: nsenter -t {{ ctninfo.container.State.Pid }} -n ip link show | grep -E eth[0-9]+ | wc -l - register: fp_num - delegate_to: "{{ VM_host[0] }}" - become: yes - -- debug: msg="{{ fp_num }}" - -- name: Set EOS backplane port name - set_fact: bp_ifname="Ethernet{{ fp_num.stdout|int - 1}}" - - name: create directory for sonic config become: yes file: @@ -27,37 +7,21 @@ - name: create config db become: yes - template: src="configdb-{{ topo }}-{{ props.swrole }}.j2" - dest="/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/sonic/config_db.json" - delegate_to: "{{ VM_host[0] }}" - -- name: create directory for frr config - become: yes - file: - path: "/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/frr" - state: directory - delegate_to: "{{ VM_host[0] }}" - -- name: create frr bgpd config - become: yes - template: src="frr-{{ topo }}-{{ props.swrole }}.j2" - dest="/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/frr/bgpd.conf" - delegate_to: "{{ VM_host[0] }}" - -- name: create zebra config - become: yes - template: src="zebra-{{ topo }}-{{ props.swrole }}.j2" - dest="/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/frr/zebra.conf" - delegate_to: "{{ VM_host[0] }}" - -- name: create frr daemons config - become: yes - template: src="frr-daemons" - dest="/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/frr/daemons" - delegate_to: "{{ VM_host[0] }}" - -- name: create vtysh config - become: yes - template: src="frr-vtysh.conf" - dest="/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/frr/vtysh.conf" - delegate_to: "{{ VM_host[0] }}" + template: + src: "{{ lookup('first_found', _csonic_configdb_candidates) }}" + dest: "/{{ csonic_image_mount_dir }}/csonic_{{ vm_set_name }}_{{ inventory_hostname }}/sonic/config_db.json" + vars: + _csonic_configdb_candidates: + files: + - "configdb-{{ topo }}-{{ props.swrole | default('') }}.j2" + - "configdb-{{ props.swrole | default('') }}.j2" + - "configdb-csonic.j2" + paths: + - "{{ role_path }}/templates" + delegate_to: "{{ VM_host[0] }}" + +# NOTE: cSONiC FRR configuration is generated by bgpcfgd from CONFIG_DB +# (BGP_NEIGHBOR entries in config_db.json), the same path as production SONiC. +# Only the /sonic directory is bind-mounted into the container; an frr/ +# directory would not be consumed, so no FRR/zebra/daemons/vtysh files are +# rendered here. diff --git a/ansible/roles/sonic/templates/configdb-csonic.j2 b/ansible/roles/sonic/templates/configdb-csonic.j2 new file mode 100644 index 00000000000..4437dbab20a --- /dev/null +++ b/ansible/roles/sonic/templates/configdb-csonic.j2 @@ -0,0 +1,186 @@ +{% set host = configuration[hostname] %} +{% set _role_type_map = {'leaf': 'LeafRouter', 'spine': 'SpineRouter', 'tor': 'ToRRouter', 'core': 'SpineRouter'} %} +{% set device_type = _role_type_map.get(props.swrole | default('leaf'), 'LeafRouter') %} +{# Count front-panel Ethernet interfaces so the backplane can be placed past #} +{# them (Ethernet{K+1}). For single-link neighbors K==1 -> Ethernet2 (same as #} +{# before); for multi-link LAG neighbors (T1-LAG/T2) the backplane moves up so #} +{# it never collides with the second front-panel port. #} +{% set _fp_eths = [] %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Ethernet') %}{% if _fp_eths.append(name) %}{% endif %}{% endif %} +{% endfor %} +{% set _bp_num = (_fp_eths | length) + 1 %} +{% set _bp_eth = 'Ethernet' ~ _bp_num %} +{# Lane numbering matches the SONiC-VM hwsku lanemap: Ethernet (1-based) #} +{# occupies 4 synthetic lanes starting at (N-1)*4 + 25 (the first usable VS #} +{# lane). These lanes are not real hardware serdes; they only need to be unique #} +{# and consistent with the image's lanemap.ini. #} +{% set _bp_lane0 = (_bp_num - 1) * 4 + 25 %} +{ + "PORT": { +{% set port_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Ethernet') %} +{{ port_comma() }} + "{{ name }}": { + "admin_status": "up", + "alias": "fortyGigE0/{{ (name | replace('Ethernet','') | int - 1) * 4 }}", + "lanes": "{{ (name | replace('Ethernet','') | int - 1) * 4 + 25 }},{{ (name | replace('Ethernet','') | int - 1) * 4 + 26 }},{{ (name | replace('Ethernet','') | int - 1) * 4 + 27 }},{{ (name | replace('Ethernet','') | int - 1) * 4 + 28 }}", + "mtu": "9100", + "speed": "40000" + } +{% endif %} +{% endfor %} +{% if host['bp_interface'] is defined %} +, + "{{ _bp_eth }}": { + "admin_status": "up", + "alias": "fortyGigE0/{{ (_bp_num - 1) * 4 }}", + "lanes": "{{ _bp_lane0 }},{{ _bp_lane0 + 1 }},{{ _bp_lane0 + 2 }},{{ _bp_lane0 + 3 }}", + "mtu": "9100", + "speed": "40000" + } +{% endif %} + }, + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "{{ host['bgp']['asn'] }}", + "hostname": "{{ hostname }}", + "type": "{{ device_type }}", + "hwsku": "SONiC-VM", + "platform": "x86_64-kvm_x86_64-r0" + } + }, + "LOOPBACK_INTERFACE": { +{% set loopback_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Loopback') %} +{{ loopback_comma() }} + "{{ name }}": {} +{% if iface['ipv4'] is defined %} +{{ loopback_comma() }} + "{{ name }}|{{ iface['ipv4'] }}": {} +{% endif %} +{% if iface['ipv6'] is defined %} +{{ loopback_comma() }} + "{{ name }}|{{ iface['ipv6'] }}": {} +{% endif %} +{% endif %} +{% endfor %} + }, +{% set has_portchannel = [] %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Port-Channel') %} +{% if has_portchannel.append(1) %}{% endif %} +{% endif %} +{% endfor %} +{% if has_portchannel %} + "PORTCHANNEL": { +{% set pc_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Port-Channel') %} +{% set pc_name = 'PortChannel' ~ name.split('Port-Channel')[1] %} +{{ pc_comma() }} + "{{ pc_name }}": { + "admin_status": "up", + "mtu": "9100", + "min_links": "1", + "lacp_key": "auto" + } +{% endif %} +{% endfor %} + }, + "PORTCHANNEL_MEMBER": { +{% set pcm_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Ethernet') and iface is mapping and 'lacp' in iface %} +{% set pc_name = 'PortChannel' ~ iface['lacp'] %} +{{ pcm_comma() }} + "{{ pc_name }}|{{ name }}": {} +{% endif %} +{% endfor %} + }, + "PORTCHANNEL_INTERFACE": { +{% set pci_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Port-Channel') %} +{% set pc_name = 'PortChannel' ~ name.split('Port-Channel')[1] %} +{{ pci_comma() }} + "{{ pc_name }}": {} +{% if iface['ipv4'] is defined %} +{{ pci_comma() }} + "{{ pc_name }}|{{ iface['ipv4'] }}": {} +{% endif %} +{% if iface['ipv6'] is defined %} +{{ pci_comma() }} + "{{ pc_name }}|{{ iface['ipv6'] }}": {} +{% endif %} +{% endif %} +{% endfor %} + }, +{% endif %} + "INTERFACE": { +{% set iface_comma = joiner(",") %} +{% for name, iface in host['interfaces'].items() %} +{% if name.startswith('Ethernet') and (iface is not mapping or 'lacp' not in iface) %} +{{ iface_comma() }} + "{{ name }}": {} +{% if iface is mapping and iface.get('ipv4') %} +{{ iface_comma() }} + "{{ name }}|{{ iface['ipv4'] }}": {} +{% endif %} +{% if iface is mapping and iface.get('ipv6') %} +{{ iface_comma() }} + "{{ name }}|{{ iface['ipv6'] }}": {} +{% endif %} +{% endif %} +{% endfor %} +{% if host['bp_interface'] is defined %} +{{ iface_comma() }} + "{{ _bp_eth }}": {} +{% if host['bp_interface']['ipv4'] is defined %} +{{ iface_comma() }} + "{{ _bp_eth }}|{{ host['bp_interface']['ipv4'] }}": {} +{% endif %} +{% if host['bp_interface']['ipv6'] is defined %} +{{ iface_comma() }} + "{{ _bp_eth }}|{{ host['bp_interface']['ipv6'] }}": {} +{% endif %} +{% endif %} + }, + "BGP_NEIGHBOR": { +{% set bgp_comma = joiner(",") %} +{% for asn, remote_ips in host['bgp']['peers'].items() %} +{% for remote_ip in remote_ips %} +{{ bgp_comma() }} + "{{ remote_ip }}": { + "asn": "{{ asn }}", + "name": "{{ asn }}", + "admin_status": "up", + "holdtime": "10", + "keepalive": "3" + } +{% endfor %} +{% endfor %} +{% if props.nhipv4 is defined and props.nhipv4 %} +{{ bgp_comma() }} + "{{ props.nhipv4 }}": { + "asn": "{{ host['bgp']['asn'] }}", + "name": "exabgp_v4", + "admin_status": "up", + "holdtime": "10", + "keepalive": "3" + } +{% endif %} +{% if props.nhipv6 is defined and props.nhipv6 %} +{{ bgp_comma() }} + "{{ props.nhipv6 }}": { + "asn": "{{ host['bgp']['asn'] }}", + "name": "exabgp_v6", + "admin_status": "up", + "holdtime": "10", + "keepalive": "3" + } +{% endif %} + } +} diff --git a/ansible/roles/sonic/templates/frr-daemons b/ansible/roles/sonic/templates/frr-daemons deleted file mode 100644 index a191e1e9c2a..00000000000 --- a/ansible/roles/sonic/templates/frr-daemons +++ /dev/null @@ -1,44 +0,0 @@ -# This file tells the FRR package which daemons to start. -# -# Entries are in the format: =(yes|no|priority) -# 0, "no" = disabled -# 1, "yes" = highest priority -# 2 .. 10 = lower priorities -# -zebra=yes -bgpd=yes -ospfd=no -ospf6d=no -ripd=no -ripngd=no -isisd=no -pimd=no -ldpd=no -nhrpd=no -eigrpd=no -babeld=no -sharpd=no -pbrd=no -bfdd=no -fabricd=no -vrrpd=no - -vtysh_enable=yes -zebra_options=" -A 127.0.0.1 -s 90000000" -bgpd_options=" -A 127.0.0.1" -ospfd_options=" -A 127.0.0.1" -ospf6d_options=" -A ::1" -ripd_options=" -A 127.0.0.1" -ripngd_options=" -A ::1" -isisd_options=" -A 127.0.0.1" -pimd_options=" -A 127.0.0.1" -ldpd_options=" -A 127.0.0.1" -nhrpd_options=" -A 127.0.0.1" -eigrpd_options=" -A 127.0.0.1" -babeld_options=" -A 127.0.0.1" -sharpd_options=" -A 127.0.0.1" -pbrd_options=" -A 127.0.0.1" -staticd_options="-A 127.0.0.1" -bfdd_options=" -A 127.0.0.1" -fabricd_options="-A 127.0.0.1" -vrrpd_options=" -A 127.0.0.1" diff --git a/ansible/roles/sonic/templates/frr-t0-leaf.j2 b/ansible/roles/sonic/templates/frr-t0-leaf.j2 deleted file mode 100644 index 241c25eae78..00000000000 --- a/ansible/roles/sonic/templates/frr-t0-leaf.j2 +++ /dev/null @@ -1,42 +0,0 @@ -{% set host = configuration[hostname] %} -! -hostname {{ hostname }} -password zebra -enable password zebra -! -log syslog informational -log facility local4 -! -router bgp {{ host['bgp']['asn'] }} - bgp router-id {{ host['interfaces']['Loopback0']['ipv4'] | ipaddr('address') }} - ! -{% for asn, remote_ips in host['bgp']['peers'].items() %} -{% for remote_ip in remote_ips %} - neighbor {{ remote_ip }} remote-as {{ asn }} - neighbor {{ remote_ip }} description {{ asn }} -{% if remote_ip | ipv6 %} - address-family ipv6 unicast - neighbor {{ remote_ip }} activate - exit -{% endif %} -{% endfor %} -{% endfor %} - neighbor {{ props.nhipv4 }} remote-as {{ host['bgp']['asn'] }} - neighbor {{ props.nhipv4 }} description exabgp_v4 - neighbor {{ props.nhipv6 }} remote-as {{ host['bgp']['asn'] }} - neighbor {{ props.nhipv6 }} description exabgp_v6 - address-family ipv6 - neighbor {{ props.nhipv6 }} activate - exit - ! -{% for name, iface in host['interfaces'].items() if name.startswith('Loopback') %} -{% if iface['ipv4'] is defined %} - address-family ipv4 unicast - network {{ iface['ipv4'] }} -{% endif %} -{% if iface['ipv6'] is defined %} - address-family ipv6 unicast - network {{ iface['ipv6'] }} -{% endif %} -{% endfor %} -! diff --git a/ansible/roles/sonic/templates/frr-vtysh.conf b/ansible/roles/sonic/templates/frr-vtysh.conf deleted file mode 100644 index e0ab9cb6f31..00000000000 --- a/ansible/roles/sonic/templates/frr-vtysh.conf +++ /dev/null @@ -1 +0,0 @@ -service integrated-vtysh-config diff --git a/ansible/roles/sonic/templates/tests/test_configdb_csonic.py b/ansible/roles/sonic/templates/tests/test_configdb_csonic.py new file mode 100644 index 00000000000..50d9bd1bacc --- /dev/null +++ b/ansible/roles/sonic/templates/tests/test_configdb_csonic.py @@ -0,0 +1,180 @@ +"""Unit tests for the cSONiC CONFIG_DB Jinja2 template (configdb-csonic.j2). + +These tests render the template with synthetic neighbor configurations and +assert on the produced CONFIG_DB JSON. They exercise the logic the template +gets wrong most easily and that has no other coverage: + + * the output is always valid JSON (comma/joiner handling), + * exabgp next-hop (props.nhipv4 / props.nhipv6) entries are emitted only when + defined and non-empty, and never leave a dangling comma when absent, + * the backplane port is placed at Ethernet{K+1} past the front-panel links + (so multi-link LAG neighbors do not collide with a second front-panel port), + * routed Ethernet interfaces land in INTERFACE while LAG members land in + PORTCHANNEL_MEMBER, and + * the synthetic VS lane numbering matches the SONiC-VM lanemap. + +The test depends only on jinja2 so it runs standalone: + + python3 ansible/roles/sonic/templates/tests/test_configdb_csonic.py + +or under pytest: + + pytest ansible/roles/sonic/templates/tests/test_configdb_csonic.py +""" +import json +import os + +from jinja2 import Environment, FileSystemLoader + +TEMPLATES_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEMPLATE_NAME = "configdb-csonic.j2" + + +def _base_host(): + """A neighbor with two front-panel links: Ethernet1 is a LAG member of + Port-Channel1, Ethernet2 is a routed interface. A backplane is present.""" + return { + "interfaces": { + "Loopback0": {"ipv4": "100.1.0.29/32", "ipv6": "2064:100::1d/128"}, + "Ethernet1": {"lacp": 1}, + "Ethernet2": {"ipv4": "10.0.0.5/31", "ipv6": "fc00::5/126"}, + "Port-Channel1": {"ipv4": "10.0.0.1/31", "ipv6": "fc00::1/126"}, + }, + "bgp": { + "asn": 64001, + "peers": {64600: ["10.0.0.0", "fc00::0"]}, + }, + "bp_interface": {"ipv4": "10.10.246.29/24", "ipv6": "fc0a::1d/64"}, + } + + +def _render(host, props): + env = Environment( + loader=FileSystemLoader(TEMPLATES_DIR), + trim_blocks=False, + lstrip_blocks=False, + ) + template = env.get_template(TEMPLATE_NAME) + hostname = "ARISTA01T1" + return template.render( + configuration={hostname: host}, + hostname=hostname, + props=props, + ) + + +def _render_json(host, props): + rendered = _render(host, props) + # A malformed template (e.g. trailing comma) makes this raise. + return json.loads(rendered) + + +def test_renders_valid_json_with_nexthops(): + cfg = _render_json(_base_host(), {"swrole": "leaf", + "nhipv4": "10.10.246.254", + "nhipv6": "fc0a::ff"}) + assert "BGP_NEIGHBOR" in cfg + # bgp peers + both exabgp next hops are present. + assert "10.0.0.0" in cfg["BGP_NEIGHBOR"] + assert "10.10.246.254" in cfg["BGP_NEIGHBOR"] + assert "fc0a::ff" in cfg["BGP_NEIGHBOR"] + + +def test_valid_json_without_nexthops(): + """Topologies that do not define nhipv4/nhipv6 must still render valid JSON + (no undefined-variable blow-up, no dangling comma).""" + cfg = _render_json(_base_host(), {"swrole": "leaf"}) + nbrs = cfg["BGP_NEIGHBOR"] + # Only the real bgp peers remain; no empty/None next-hop key sneaks in. + assert set(nbrs.keys()) == {"10.0.0.0", "fc00::0"} + assert "" not in nbrs + assert "None" not in nbrs + + +def test_empty_nexthop_is_skipped(): + """An empty-string next hop must be treated like an absent one.""" + cfg = _render_json(_base_host(), {"swrole": "leaf", + "nhipv4": "", + "nhipv6": None}) + assert set(cfg["BGP_NEIGHBOR"].keys()) == {"10.0.0.0", "fc00::0"} + + +def test_only_v4_nexthop(): + cfg = _render_json(_base_host(), {"swrole": "leaf", + "nhipv4": "10.10.246.254"}) + nbrs = cfg["BGP_NEIGHBOR"] + assert "10.10.246.254" in nbrs + assert all(":" not in k or k in ("fc00::0",) for k in nbrs) + assert set(nbrs.keys()) == {"10.0.0.0", "fc00::0", "10.10.246.254"} + + +def test_no_bgp_peers_only_nexthops(): + """No real peers, only exabgp next hops: still valid JSON, no leading comma.""" + host = _base_host() + host["bgp"]["peers"] = {} + cfg = _render_json(host, {"swrole": "leaf", + "nhipv4": "10.10.246.254", + "nhipv6": "fc0a::ff"}) + assert set(cfg["BGP_NEIGHBOR"].keys()) == {"10.10.246.254", "fc0a::ff"} + + +def test_backplane_placed_after_front_panel_links(): + """With K=2 front-panel Ethernet ports, the backplane is Ethernet3.""" + cfg = _render_json(_base_host(), {"swrole": "leaf", + "nhipv4": "10.10.246.254", + "nhipv6": "fc0a::ff"}) + assert "Ethernet3" in cfg["PORT"] + assert "Ethernet3" in cfg["INTERFACE"] + assert "Ethernet3|10.10.246.29/24" in cfg["INTERFACE"] + # Single-link neighbor keeps the backplane at Ethernet2. + single = _base_host() + del single["interfaces"]["Ethernet2"] + del single["interfaces"]["Port-Channel1"] + single["interfaces"]["Ethernet1"] = {"ipv4": "10.0.0.5/31"} + cfg1 = _render_json(single, {"swrole": "leaf"}) + assert "Ethernet2" in cfg1["PORT"] + + +def test_routed_vs_lag_member_split(): + cfg = _render_json(_base_host(), {"swrole": "leaf"}) + # Routed Ethernet2 gets an INTERFACE entry with its IPs. + assert "Ethernet2" in cfg["INTERFACE"] + assert "Ethernet2|10.0.0.5/31" in cfg["INTERFACE"] + # LAG member Ethernet1 is a PORTCHANNEL_MEMBER, not a routed INTERFACE. + assert "PortChannel1|Ethernet1" in cfg["PORTCHANNEL_MEMBER"] + assert "Ethernet1" not in cfg["INTERFACE"] + + +def test_port_lane_numbering(): + cfg = _render_json(_base_host(), {"swrole": "leaf"}) + # Ethernet2 -> (2-1)*4 + 25 = 29 -> "29,30,31,32". + assert cfg["PORT"]["Ethernet2"]["lanes"] == "29,30,31,32" + # Ethernet1 -> (1-1)*4 + 25 = 25 -> "25,26,27,28". + assert cfg["PORT"]["Ethernet1"]["lanes"] == "25,26,27,28" + + +def test_no_backplane_when_absent(): + host = _base_host() + del host["bp_interface"] + cfg = _render_json(host, {"swrole": "leaf"}) + assert "Ethernet3" not in cfg["PORT"] + + +def _run_standalone(): + tests = [v for k, v in sorted(globals().items()) + if k.startswith("test_") and callable(v)] + failures = 0 + for t in tests: + try: + t() + print("PASS", t.__name__) + except Exception as exc: # noqa: BLE001 + failures += 1 + print("FAIL", t.__name__, "->", repr(exc)) + print("\n{} passed, {} failed".format(len(tests) - failures, failures)) + return failures + + +if __name__ == "__main__": + import sys + sys.exit(1 if _run_standalone() else 0) diff --git a/ansible/roles/sonic/templates/zebra-t0-leaf.j2 b/ansible/roles/sonic/templates/zebra-t0-leaf.j2 deleted file mode 100644 index b59b34c483b..00000000000 --- a/ansible/roles/sonic/templates/zebra-t0-leaf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -{% set host = configuration[hostname] %} -hostname {{ hostname }} -password zebra -enable password zebra -! -log syslog informational -log facility local4 -! -! end of template: common/daemons.common.conf.j2! -! -! -! Enable link-detect (default disabled) -{% for name, iface in host['interfaces'].items() %} -interface {{ name }} -link detect -! -{% endfor %} diff --git a/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py b/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py index 4c27fa14115..2eec7b29a61 100644 --- a/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py +++ b/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py @@ -1335,3 +1335,47 @@ def runTest(self): self.client_send_bootp() self.client_send_unknown(self.dest_mac_address, self.client_udp_src_port) self.server_send_unknown() + + +class DHCPBroadcastNotFloodedTest(DHCPTest): + """ + Test that DHCP broadcast packets (Discover, Request) are trapped to CPU + by the COPP trap rule and NOT L2-flooded to other VLAN member ports. + + The COPP policy configures trap_action=trap (SAI_PACKET_ACTION_TRAP) for + DHCP, which means the packet is removed from the forwarding pipeline and + sent exclusively to CPU. This test verifies the non-flooding behavior by + sending DHCP broadcast packets from one client port and asserting that + zero copies of the original broadcast appear on other VLAN member ports. + """ + + def verify_broadcast_not_flooded(self, pkt, packet_type): + """Send a DHCP broadcast packet and verify it is NOT flooded to other VLAN member ports.""" + if 'other_client_port' not in self.test_params or not self.other_client_port: + logger.warning("No other client ports configured, skipping non-flood check for %s", packet_type) + return + + logger.info("Sending %s from client port %d", packet_type, self.client_port_index) + testutils.send_packet(self, self.client_port_index, pkt) + + # Build a mask for the original broadcast packet to match against + masked_pkt = Mask(pkt) + masked_pkt.set_do_not_care_scapy(scapy.Ether, "src") + self.set_common_ignored_mask_fields(masked_pkt) + + # Negative check: original broadcast must NOT appear on other VLAN member ports + flooded_count = testutils.count_matched_packets_all_ports( + self, masked_pkt, self.other_client_port, timeout=3) + self.assertTrue(flooded_count == 0, + "Failed: %s broadcast was flooded to %d other VLAN member port(s), expected 0" + % (packet_type, flooded_count)) + logger.info("%s broadcast was correctly NOT flooded to other VLAN member ports", packet_type) + + def runTest(self): + # Verify DHCP Discover broadcast is not flooded + dhcp_discover = self.create_dhcp_discover_packet(self.BROADCAST_MAC, self.DHCP_CLIENT_PORT) + self.verify_broadcast_not_flooded(dhcp_discover, "Discover") + + # Verify DHCP Request broadcast is not flooded + dhcp_request = self.create_dhcp_request_packet(self.BROADCAST_MAC, self.DHCP_CLIENT_PORT) + self.verify_broadcast_not_flooded(dhcp_request, "Request") diff --git a/ansible/roles/test/files/ptftests/py3/hash_test.py b/ansible/roles/test/files/ptftests/py3/hash_test.py index 3bb9b721c29..db4aac7acf7 100644 --- a/ansible/roles/test/files/ptftests/py3/hash_test.py +++ b/ansible/roles/test/files/ptftests/py3/hash_test.py @@ -39,7 +39,8 @@ class HashTest(BaseTest): # Class variables # --------------------------------------------------------------------- DEFAULT_BALANCING_RANGE = 0.25 - RELAXED_BALANCING_RANGE = 0.80 + RELAXED_BALANCING_RANGE = 0.8 + RELAXED_BALANCING_RANGE_MAXTOPO = 1.5 BALANCING_TEST_TIMES = 250 DEFAULT_SWITCH_TYPE = 'voq' _required_params = [ @@ -543,8 +544,10 @@ def check_within_expected_range(self, actual, expected, hash_key): return (percentage, actual >= expected * 0.2) elif 't2' in self.topo_name: # ip-protocol only has 8-bits of entropy which results in poor hashing distributions on topologies with - # a large number of ecmp paths so relax the hashing requirements - balancing_range = self.RELAXED_BALANCING_RANGE + # a large number of ecmp paths so relax the hashing requirements. For max port count topologies, + # relax the hashing requirements further + balancing_range = self.RELAXED_BALANCING_RANGE_MAXTOPO if "max" in self.topo_name \ + else self.RELAXED_BALANCING_RANGE return (percentage, abs(percentage) <= balancing_range) def check_same_asic(self, src_port, exp_port_list): diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt index d37fed1aed2..cba2d5a43cd 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt @@ -508,7 +508,7 @@ r, ".* ERR swss\d*#orchagent:.*getPortLinkTrainingFailure: Failed to get LT fail # https://github.com/sonic-net/sonic-mgmt/issues/24023 r, ".* ERR syncd\d*#syncd:.*SAI_API_PORT:brcm_sai_get_port_attribute_cmn:\d+ RX signal detect status get \d+ attrib \d+ failed with error Feature unavailable \(0xfffffff0\).*" r, ".* ERR syncd\d*#syncd:.*SAI_API_PORT:brcm_sai_get_port_attribute_cmn:\d+ bcm_port_phy_control_get rx snr failed with error Feature unavailable \(0xfffffff0\).*" -r, ".* ERR syncd\d*#syncd: :- collectData: Failed to get port attr for VID 0x[0-9a-fA-F]+, RID:0x[0-9a-fA-F]+: -2.*" +r, ".* ERR syncd\d*#syncd: :- collectData: Failed to get port attr for VID 0x[0-9a-fA-F]+, RID:0x[0-9a-fA-F]+: -\d+.*" # https://github.com/sonic-net/sonic-mgmt/issues/19347 r, ".* ERR auditd.*: queue to plugins is full - dropping event" diff --git a/ansible/roles/test/tasks/ecmp.yml b/ansible/roles/test/tasks/ecmp.yml index cb1dc49db4c..8d23a2f96d0 100644 --- a/ansible/roles/test/tasks/ecmp.yml +++ b/ansible/roles/test/tasks/ecmp.yml @@ -26,7 +26,7 @@ - 'address-family ipv4' - 'network 100.1.1.1/32' - 'quit' - when: "{{ ipv6 }} == False" + when: not (ipv6 | bool) - name: Initialize the array with the list of commands to be sent to the VM set_fact: @@ -40,7 +40,7 @@ - 'address-family ipv6' - 'network 2064:200::1/128' - 'quit' - when: "{{ ipv6 }} == True" + when: ipv6 | bool - name: Configure VMs to add loopback interface configure_vms: ip={{ item }} cmds={{ commands }} diff --git a/ansible/roles/test/tasks/ecmp/link_down.yml b/ansible/roles/test/tasks/ecmp/link_down.yml index afb0bfdf325..a2107af00c8 100644 --- a/ansible/roles/test/tasks/ecmp/link_down.yml +++ b/ansible/roles/test/tasks/ecmp/link_down.yml @@ -43,11 +43,11 @@ - set_fact: config_cmd: ifconfig -a | grep -A 1 "lo0" | tail -1 | xargs | cut -d" " -f2 | cut -d":" -f2 - when: "{{ ipv6 }} == False" + when: not (ipv6 | bool) - set_fact: config_cmd: ifconfig -a | grep -A 2 "lo0" | tail -1 | xargs | cut -d" " -f3 | cut -d"/" -f1 - when: "{{ ipv6 }} == True" + when: ipv6 | bool - name: Ping destination VM from host VM. shell: "{{ config_cmd }}" @@ -71,11 +71,11 @@ - set_fact: ping_cmd: ping -n -i 0.004 {{ loopback_ip.stdout }} -c 30000 | grep "received" | cut -d"," -f2 | xargs | cut -d" " -f1 - when: "{{ ipv6 }} == False" + when: not (ipv6 | bool) - set_fact: ping_cmd: ping6 {{ loopback_ip.stdout }} -i 0.004 -c 30000 | grep "received" | cut -d"," -f2 | xargs | cut -d" " -f1 - when: "{{ ipv6 }} == True" + when: ipv6 | bool - name: Ping destination VM from host VM. shell: "{{ ping_cmd }}" diff --git a/ansible/roles/test/vars/testcases.yml b/ansible/roles/test/vars/testcases.yml index 4adb1d4c90b..9f21c95517b 100644 --- a/ansible/roles/test/vars/testcases.yml +++ b/ansible/roles/test/vars/testcases.yml @@ -49,7 +49,7 @@ testcases: continuous_reboot: filename: continuous_reboot.yml vtestbed_compatible: no - topologies: [t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-118, t0-120, t0-88-o8c80, t1, t1-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, t1-28-lag, t1-48-lag, t0-isolated-d16u16s1, t1-isolated-d28u1, t0-isolated-d32u32s2, t0-isolated-v6-d32u32s2, t1-isolated-d56u2, t1-isolated-v6-d56u1-lag, t0-isolated-d256u256s2, t1-isolated-d448u15-lag] + topologies: [t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-118, t0-120, t0-88-o8c80, t1, t1-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, t1-28-lag, t1-48-lag, t0-isolated-d16u16s1, t1-isolated-d28u1, t0-isolated-d32u32s2, t0-isolated-d32u32s2-mix, t0-isolated-v6-d32u32s2, t1-isolated-d56u2, t1-isolated-v6-d56u1-lag, t0-isolated-d256u256s2, t1-isolated-d448u15-lag] copp: @@ -260,7 +260,7 @@ testcases: reboot: filename: reboot.yml - topologies: [dualtor, dualtor-64-breakout, dualtor-aa-64-breakout, t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-118, t0-120, t0-88-o8c80, t1, t1-lag, t1-28-lag, t1-48-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, ptf32, ptf64, t0-isolated-d16u16s1, t1-isolated-d28u1, t0-isolated-d32u32s2, t0-isolated-v6-d32u32s2, t1-isolated-d56u2, t1-isolated-v6-d56u1-lag, t0-isolated-d256u256s2, t1-isolated-d448u15-lag] + topologies: [dualtor, dualtor-64-breakout, dualtor-aa-64-breakout, t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-118, t0-120, t0-88-o8c80, t1, t1-lag, t1-28-lag, t1-48-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, ptf32, ptf64, t0-isolated-d16u16s1, t1-isolated-d28u1, t0-isolated-d32u32s2, t0-isolated-d32u32s2-mix, t0-isolated-v6-d32u32s2, t1-isolated-d56u2, t1-isolated-v6-d56u1-lag, t0-isolated-d256u256s2, t1-isolated-d448u15-lag] repeat_harness: filename: repeat_harness.yml diff --git a/ansible/roles/testbed/nut/tasks/l1_apply_cross_connects.yml b/ansible/roles/testbed/nut/tasks/l1_apply_cross_connects.yml new file mode 100644 index 00000000000..4490dc90362 --- /dev/null +++ b/ansible/roles/testbed/nut/tasks/l1_apply_cross_connects.yml @@ -0,0 +1,47 @@ +--- +# Reconcile OCS cross-connects on an L1 (FanoutL1Sonic) switch using the live CLI, +# instead of building a JSON patch and reloading the config. +# +# All of the parsing and set math lives in the ocs_cross_connect Python module +# (ansible/library/ocs_cross_connect.py); this task is a single declarative call. +# The reconcile is idempotent. A "config save" step is run after successful changes +# so cross-connect updates are explicitly written to config DB. +# +# Tunables (override with -e): +# l1_xconnect_dry_run (default false): only compute the planned changes, do not apply. +# l1_xconnect_clear_stale (default true): clear status-only entries that block a desired port. +# l1_xconnect_verify (default true): verify status is "tuned" after applying. +# l1_xconnect_save_config (default true): run "config save -y" after applying changes. +# l1_xconnect_verify_retries / l1_xconnect_verify_delay: status polling controls. +# ocs_xconn_show_config_cmd / ocs_xconn_show_status_cmd: show command overrides. +# ocs_xconn_add_cmd / ocs_xconn_del_cmd: command templates with an "{id}" placeholder. + +- name: configure OCS cross-connects via live CLI + become: true + check_mode: "{{ l1_xconnect_dry_run | default(false) | bool }}" + ocs_cross_connect: + cross_connects: "{{ device_l1_cross_connects[inventory_hostname] | default({}) }}" + clear_stale: "{{ l1_xconnect_clear_stale | default(true) | bool }}" + verify: "{{ l1_xconnect_verify | default(true) | bool }}" + verify_retries: "{{ l1_xconnect_verify_retries | default(12) | int }}" + verify_delay: "{{ l1_xconnect_verify_delay | default(5) | int }}" + register: ocs_reconcile + +- name: show OCS cross-connect progress + debug: + msg: + - "desired={{ ocs_reconcile.desired | default([]) | length }}" + - "cleared_stale={{ ocs_reconcile.cleared_stale | default([]) | length }}" + - "removed={{ ocs_reconcile.removed | default([]) | length }}" + - "added={{ ocs_reconcile.added | default([]) | length }}" + - "cleared_stale_ids: {{ ocs_reconcile.cleared_stale | default([]) }}" + - "removed_ids: {{ ocs_reconcile.removed | default([]) }}" + - "added_ids: {{ ocs_reconcile.added | default([]) }}" + +- name: save config DB after OCS cross-connect apply + become: true + command: config save -y + when: + - ocs_reconcile is changed + - l1_xconnect_save_config | default(true) | bool + - not (l1_xconnect_dry_run | default(false) | bool) diff --git a/ansible/roles/testbed/nut/tasks/main.yml b/ansible/roles/testbed/nut/tasks/main.yml index 7e6ccab65b5..cbdbca71faa 100644 --- a/ansible/roles/testbed/nut/tasks/main.yml +++ b/ansible/roles/testbed/nut/tasks/main.yml @@ -6,17 +6,23 @@ device_info[inventory_hostname] is defined and device_info[inventory_hostname].Type == 'FanoutL1Sonic' block: - - import_tasks: l1_create_config_patch.yml - - import_tasks: device_prepare_config.yml - - import_tasks: device_apply_config.yml - when: deploy is defined and deploy|bool == true + - name: deploy L1 config via patch and reload + when: not (use_l1_ocs_cli | default(false) | bool) + block: + - import_tasks: l1_create_config_patch.yml + - import_tasks: device_prepare_config.yml + - import_tasks: device_apply_config.yml + when: deploy is defined and deploy|bool == true + - name: deploy L1 OCS cross-connects via live CLI + import_tasks: l1_apply_cross_connects.yml + when: use_l1_ocs_cli | default(false) | bool - name: config duts when: (config_duts is not defined or config_duts|bool == true) and device_info[inventory_hostname] is defined and device_info[inventory_hostname].Type != 'FanoutL1Sonic' block: - - import_tasks: dut_create_config_patch.yml - - import_tasks: device_prepare_config.yml - - import_tasks: device_apply_config.yml - when: deploy is defined and deploy|bool == true + - import_tasks: dut_create_config_patch.yml + - import_tasks: device_prepare_config.yml + - import_tasks: device_apply_config.yml + when: deploy is defined and deploy|bool == true diff --git a/ansible/roles/vm_set/library/csonic_network.py b/ansible/roles/vm_set/library/csonic_network.py index fa7cf6272a3..41651d8554c 100644 --- a/ansible/roles/vm_set/library/csonic_network.py +++ b/ansible/roles/vm_set/library/csonic_network.py @@ -226,7 +226,7 @@ class CsonicNetwork(object): """ def __init__(self, ctn_name, vm_name, mgmt_br_name, fp_mtu, max_fp_num, - vm_offset=0, sonic_naming=True, bp_bridge=None): + vm_offset=0, sonic_naming=True, bp_bridge=None, num_fp_links=1): self.ctn_name = ctn_name self.vm_name = vm_name self.fp_mtu = fp_mtu @@ -235,6 +235,10 @@ def __init__(self, ctn_name, vm_name, mgmt_br_name, fp_mtu, max_fp_num, self.mgmt_br_name = mgmt_br_name self.sonic_naming = sonic_naming self.bp_bridge = bp_bridge + # Number of front-panel links this neighbor has (== number of topology + # vlans for the VM). Multi-link (LAG) neighbors on T1-LAG/T2 need one + # front-panel interface per link. + self.num_fp_links = max(1, int(num_fp_links)) self.pid = CsonicNetwork.get_pid(self.ctn_name) if self.pid is None: @@ -245,42 +249,57 @@ def init_network(self): This creates veth pairs injected into the container: - eth0: management interface, connected to mgmt bridge - - eth1: front panel interface, connected to OVS bridge - - eth2: backplane interface, for PTF/ExaBGP connectivity - - The sonic-vs start.sh maps eth1 -> Ethernet0 via lanemap.ini. - Each cSONiC VM represents one neighbor with a single front panel port. + - eth1..ethK: front panel interfaces (one per topology vlan/link), + each connected to its OVS bridge br-- + - eth{K+1}: backplane interface, for PTF/ExaBGP connectivity + + The sonic-vs start.sh maps eth -> Ethernet via lanemap.ini, so a + front-panel link index i maps to Ethernet(i+1) in CONFIG_DB. Placing the + backplane on eth{K+1} keeps it clear of the front-panel ports even for + multi-link (LAG) neighbors (T1-LAG/T2), where K can be > 1. """ + num_fp = self.num_fp_links + # Create management link (eth0) mp_name = MGMT_TAP_TEMPLATE % (self.vm_name) self.add_veth_if_to_docker(mp_name, TMP_TAP_TEMPLATE % ( self.vm_name, 0), INT_TAP_TEMPLATE % 0) self.add_if_to_bridge(mp_name, self.mgmt_br_name) - # Create front panel link (Ethernet0) - # Each cSONiC VM has Ethernet0 as its front panel port, connected to - # the OVS bridge br--0 which links to the corresponding DUT port - fp_name = FP_TAP_TEMPLATE % (self.vm_name, 0) # VM0100-t0 - fp_br_name = OVS_FP_BRIDGE_TEMPLATE % (self.vm_name, 0) # br-VM0100-0 + # Create front panel links. Front-panel link i is connected to the OVS + # bridge br-- which links to the corresponding DUT port, and + # appears inside the container as eth(i+1) -> Ethernet(i+1). + # + # NOTE: the cSONiC deployment always passes sonic_naming=False (see + # add_csonic.yml): interfaces are injected as eth1..ethK and the + # container's start.sh maps eth -> Ethernet via lanemap.ini. The + # sonic_naming=True path below is legacy/experimental and names the + # netdevs Ethernet directly; it is NOT exercised by the multi-link + # wiring and its static backplane name (SONIC_BP_TEMPLATE) does not + # account for num_fp_links, so do not enable it without revisiting the + # backplane placement. + for i in range(num_fp): + fp_name = FP_TAP_TEMPLATE % (self.vm_name, i) # VM0100-t + fp_br_name = OVS_FP_BRIDGE_TEMPLATE % (self.vm_name, i) # br-VM0100- + + if self.sonic_naming: + int_if_name = SONIC_INT_TEMPLATE % i + else: + int_if_name = INT_TAP_TEMPLATE % (i + 1) # eth1..ethK - if self.sonic_naming: - int_if_name = SONIC_INT_TEMPLATE % 0 # Always Ethernet0 - else: - int_if_name = INT_TAP_TEMPLATE % 1 # eth1 + self.add_veth_if_to_docker( + fp_name, + TMP_TAP_TEMPLATE % (self.vm_name, i + 1), + int_if_name + ) + self.add_if_to_ovs_bridge(fp_name, fp_br_name) - self.add_veth_if_to_docker( - fp_name, - TMP_TAP_TEMPLATE % (self.vm_name, 1), - int_if_name - ) - self.add_if_to_ovs_bridge(fp_name, fp_br_name) - - # Create backplane link (eth_bp) - bp_int_name = SONIC_BP_TEMPLATE if self.sonic_naming else INT_TAP_TEMPLATE % 2 + # Create backplane link (eth{K+1}) past the last front-panel port. + bp_int_name = SONIC_BP_TEMPLATE if self.sonic_naming else INT_TAP_TEMPLATE % (num_fp + 1) bp_name = BP_TAP_TEMPLATE % (self.vm_name) self.add_veth_if_to_docker( bp_name, - TMP_TAP_TEMPLATE % (self.vm_name, 2), + TMP_TAP_TEMPLATE % (self.vm_name, num_fp + 1), bp_int_name) # Note: Backplane bridge connection is handled by vm_topology.py @@ -566,6 +585,7 @@ def main(): vm_offset=dict(required=False, type='int', default=0), sonic_naming=dict(required=False, type='bool', default=True), bp_bridge=dict(required=False, type='str', default=None), + num_fp_links=dict(required=False, type='int', default=1), ), supports_check_mode=False) @@ -577,11 +597,13 @@ def main(): vm_offset = module.params['vm_offset'] sonic_naming = module.params['sonic_naming'] bp_bridge = module.params['bp_bridge'] + num_fp_links = module.params['num_fp_links'] config_module_logging('csonic_net_' + vm_name) try: - cnet = CsonicNetwork(name, vm_name, mgmt_bridge, fp_mtu, max_fp_num, vm_offset, sonic_naming, bp_bridge) + cnet = CsonicNetwork(name, vm_name, mgmt_bridge, fp_mtu, max_fp_num, vm_offset, + sonic_naming, bp_bridge, num_fp_links) cnet.init_network() except Exception as error: diff --git a/ansible/roles/vm_set/library/vm_topology.py b/ansible/roles/vm_set/library/vm_topology.py index 0a9c9f83abb..b13b55360de 100644 --- a/ansible/roles/vm_set/library/vm_topology.py +++ b/ansible/roles/vm_set/library/vm_topology.py @@ -317,6 +317,7 @@ def init(self, vm_set_name, vm_base, duts_fp_ports, duts_name, ptf_exists=True, # For now distinguish a cable topology since it does not contain any vms and there are two ToR's self._is_cable = True if len( self.duts_name) > 1 and 'VMs' not in self.topo else False + self._is_smartswitch_ha = self.topo.get('topo_type') == 't1-smartswitch-ha' self.host_interfaces = self.topo.get('host_interfaces', []) if self.dut_interfaces: @@ -610,7 +611,8 @@ def add_bp_port_to_docker(self, mgmt_ip, mgmt_ipv6): self.add_ip_to_docker_if(BP_PORT_NAME, mgmt_ip, mgmt_ipv6) VMTopology.iface_disable_txoff(BP_PORT_NAME, self.pid) - def add_bp_port_with_vlans_to_docker(self, vlan_data, vrf_map, multi_vrf_config): + def add_bp_port_with_vlans_to_docker(self, vlan_data, vrf_map, multi_vrf_config, + ptf_bp_ip_addr=None, ptf_bp_ipv6_addr=None): rev_vrf_map = {} for peer, vrfs in vrf_map.items(): for vrf in vrfs: @@ -619,6 +621,13 @@ def add_bp_port_with_vlans_to_docker(self, vlan_data, vrf_map, multi_vrf_config) self.add_br_if_to_docker( self.bp_bridge, PTF_BP_IF_TEMPLATE % self.vm_set_name, BP_PORT_NAME) + # Stock-compat: also give the parent backplane the legacy + # ptf_bp_ip address (e.g. 10.10.246.254/22, fc0a::ff/64) so untagged + # backplane-based tests (e.g. bgp_stress_link_flap monitor) keep working + # on converged VRF topologies. Per-VRF VLAN sub-interfaces are added below. + if ptf_bp_ip_addr or ptf_bp_ipv6_addr: + self.add_ip_to_docker_if(BP_PORT_NAME, ptf_bp_ip_addr, ptf_bp_ipv6_addr) + for vrf, data in vlan_data.items(): vlan_id = data.get("vlan") addr = data.get("ipv4") @@ -1298,8 +1307,12 @@ def bind_ovs_ports(self, br_name, dut_iface, injected_iface, vm_iface, disconnec (br_name, dut_iface_id, vm_iface_id, injected_iface_id)) bind_helper("ovs-ofctl add-flow %s table=0,priority=6,udp6,in_port=%s,udp_dst=4784,action=output:%s" % (br_name, dut_iface_id, injected_iface_id)) - bind_helper("ovs-ofctl add-flow %s table=0,priority=5,ip,in_port=%s,action=output:%s,%s" % - (br_name, dut_iface_id, vm_iface_id, injected_iface_id)) + if self._is_smartswitch_ha: + bind_helper("ovs-ofctl add-flow %s table=0,priority=5,ip,in_port=%s,action=output:%s,%s" % + (br_name, dut_iface_id, vm_iface_id, injected_iface_id)) + else: + bind_helper("ovs-ofctl add-flow %s table=0,priority=5,ip,in_port=%s,action=output:%s" % + (br_name, dut_iface_id, injected_iface_id)) bind_helper("ovs-ofctl add-flow %s table=0,priority=5,ipv6,in_port=%s,action=output:%s,%s" % (br_name, dut_iface_id, vm_iface_id, injected_iface_id)) bind_helper("ovs-ofctl add-flow %s table=0,priority=3,in_port=%s,action=output:%s,%s" % @@ -1631,7 +1644,13 @@ def remove_veth_if_from_docker(self, ext_if, int_if, tmp_name): Remove veth interface from docker """ logging.info("=== Cleanup port, int_if: %s, ext_if: %s, tmp_name: %s ===" % (ext_if, int_if, tmp_name)) - if VMTopology.intf_exists(int_if, pid=self.pid): + # When the PTF container is absent, self.pid is None and the interface that used to live + # inside the container (e.g. eth0/mgmt/backplane) is already gone with the container. The + # in-container manipulation below must be skipped in that case: intf_exists()/iface_down() + # with pid=None fall back to the host root namespace and would operate on a same-named host + # interface (e.g. eth0), knocking the host off the network. Only the host-side peer (ext_if, + # which is uniquely named per vm_set) still needs to be cleaned up. + if self.pid is not None and VMTopology.intf_exists(int_if, pid=self.pid): # Name it back to temp name in PTF container to avoid potential conflicts VMTopology.iface_down(int_if, pid=self.pid) VMTopology.cmd("nsenter -t %s -n ip link set dev %s name %s" % (self.pid, int_if, tmp_name)) @@ -2409,7 +2428,11 @@ def main(): vlan_data = multi_vrf_data.get("ptf_backplane_addrs", {}) vrf_map = multi_vrf_data.get("convergence_mapping", {}) multi_vrf_config = multi_vrf_data.get("converged_peers", {}) - net.add_bp_port_with_vlans_to_docker(vlan_data, vrf_map, multi_vrf_config) + net.add_bp_port_with_vlans_to_docker( + vlan_data, vrf_map, multi_vrf_config, + ptf_bp_ip_addr=ptf_bp_ip_addr, + ptf_bp_ipv6_addr=ptf_bp_ipv6_addr, + ) else: net.add_bp_port_to_docker(ptf_bp_ip_addr, ptf_bp_ipv6_addr) if is_vs_chassis: @@ -2595,7 +2618,11 @@ def main(): vlan_data = multi_vrf_data.get("ptf_backplane_addrs", {}) vrf_map = multi_vrf_data.get("convergence_mapping", {}) multi_vrf_config = multi_vrf_data.get("converged_peers", {}) - net.add_bp_port_with_vlans_to_docker(vlan_data, vrf_map, multi_vrf_config) + net.add_bp_port_with_vlans_to_docker( + vlan_data, vrf_map, multi_vrf_config, + ptf_bp_ip_addr=ptf_bp_ip_addr, + ptf_bp_ipv6_addr=ptf_bp_ipv6_addr, + ) else: net.add_bp_port_to_docker(ptf_bp_ip_addr, ptf_bp_ipv6_addr) diff --git a/ansible/roles/vm_set/tasks/add_csonic.yml b/ansible/roles/vm_set/tasks/add_csonic.yml index d3a68a7288b..8fa83a6341b 100644 --- a/ansible/roles/vm_set/tasks/add_csonic.yml +++ b/ansible/roles/vm_set/tasks/add_csonic.yml @@ -2,8 +2,8 @@ become: yes docker_container: name: net_{{ vm_set_name }}_{{ vm_item }} - image: debian:jessie - pull: yes + image: "{{ csonic_netbase_image }}" + pull: "{{ csonic_netbase_image_pull }}" state: started restart: no tty: yes @@ -21,4 +21,5 @@ max_fp_num: "{{ max_fp_num }}" mgmt_bridge: "{{ mgmt_bridge }}" vm_offset: "{{ topology.VMs[vm_inv_to_topo[vm_item]].vm_offset }}" + num_fp_links: "{{ topology.VMs[vm_inv_to_topo[vm_item]].vlans | length }}" sonic_naming: false diff --git a/ansible/roles/vm_set/tasks/add_topo.yml b/ansible/roles/vm_set/tasks/add_topo.yml index 450c87d2e36..cb1c44e9447 100644 --- a/ansible/roles/vm_set/tasks/add_topo.yml +++ b/ansible/roles/vm_set/tasks/add_topo.yml @@ -353,6 +353,13 @@ become: yes when: dpu_targets is defined and dpu_targets | length > 0 + - name: Don't accept ipv6 router advertisements on PTF mgmt interface if it exists + command: > + docker exec -i ptf_{{ vm_set_name }} + sh -c 'test ! -e /proc/sys/net/ipv6/conf/mgmt/accept_ra || + sysctl -w net.ipv6.conf.mgmt.accept_ra=0' + become: yes + - name: Change MAC address for PTF interfaces include_tasks: ptf_change_mac.yml when: topo != 'fullmesh' and not (ptf_use_docker_network | default(false)) diff --git a/ansible/roles/vm_set/tasks/main.yml b/ansible/roles/vm_set/tasks/main.yml index 934aff03106..45f6839807c 100644 --- a/ansible/roles/vm_set/tasks/main.yml +++ b/ansible/roles/vm_set/tasks/main.yml @@ -257,6 +257,21 @@ topo_is_multi_vrf: False when: topo_is_multi_vrf is not defined +# Honor the converger-provided front-panel count for EVERY vm_set action, not +# just 'start'. A converged (multi-VRF) cEOS prime hosts many merged sub-peers, +# so the converger surfaces the required fp count as max_fp_num_provided at the +# topo root. The 'add_topo' action creates the br--N OVS bridges +# (add_ceos_list.yml "Create VMs network") and then binds them; without this +# override max_fp_num stays at the default 4, only 4 bridges are created, and +# the subsequent bind fails with "Too many vlans. Maximum is 4". start.yml has +# its own copy for the 'start' action; doing it here covers add_topo / +# connect_vms / renumber too. Gated on max_fp_num_provided so stock and +# non-converged topologies are byte-identical. +- name: Pick provided max num of fp + set_fact: + max_fp_num: "{{ max_fp_num_provided }}" + when: max_fp_num_provided is defined + - name: Check VM type fail: msg: "Cannot support this VM type {{ vm_type }}" diff --git a/ansible/roles/vm_set/tasks/renumber_topo.yml b/ansible/roles/vm_set/tasks/renumber_topo.yml index 854d4477de1..6d6cb9f8afa 100644 --- a/ansible/roles/vm_set/tasks/renumber_topo.yml +++ b/ansible/roles/vm_set/tasks/renumber_topo.yml @@ -32,6 +32,7 @@ vars: ptf_portchannel_action: stop when: "'bmc' not in topo" + ignore_errors: yes - name: Kill exabgp and ptf_nn_agent processes in PTF container ptf_control: @@ -141,6 +142,10 @@ command: docker exec -i ptf_{{ vm_set_name }} sysctl -w net.ipv6.route.max_size=168000 become: yes + - name: Don't accept ipv6 router advertisements for docker container ptf_{{ vm_set_name }} + command: docker exec -i ptf_{{ vm_set_name }} sysctl -w net.ipv6.conf.default.accept_ra=0 + become: yes + - name: Create file to store dut type in PTF command: docker exec -i ptf_{{ vm_set_name }} sh -c 'echo {{ hostvars[duts_name.split(',')[0]]['type'] }} > /sonic/dut_type.txt' when: @@ -195,6 +200,13 @@ become: yes when: "'bmc' not in topo" + - name: Don't accept ipv6 router advertisements on PTF mgmt interface if it exists + command: > + docker exec -i ptf_{{ vm_set_name }} + sh -c 'test ! -e /proc/sys/net/ipv6/conf/mgmt/accept_ra || + sysctl -w net.ipv6.conf.mgmt.accept_ra=0' + become: yes + - name: Change MAC address for PTF interfaces include_tasks: ptf_change_mac.yml when: topo != 'fullmesh' and not (ptf_use_docker_network | default(false)) diff --git a/ansible/roles/vm_set/tasks/start_sonic_vm.yml b/ansible/roles/vm_set/tasks/start_sonic_vm.yml index 0b792adf2e7..060f5f4056f 100644 --- a/ansible/roles/vm_set/tasks/start_sonic_vm.yml +++ b/ansible/roles/vm_set/tasks/start_sonic_vm.yml @@ -37,7 +37,7 @@ asic_type: "{{ hostvars[dut_name].asic_type | default('') }}" - set_fact: - src_disk_image: "{{ sonic_vm_storage_location }}/images/sonic-{{ 'vpp' if 'vpp' == asic_type else 'vs' }}.img" + src_disk_image: "{{ lookup('env', 'SONIC_MGMT_SONIC_KVM_IMAGE') | default(sonic_vm_storage_location ~ '/images/sonic-' ~ ('vpp' if 'vpp' == asic_type else 'vs') ~ '.img', true) }}" - name: Remove arp entry for {{ dut_name }} shell: arp -d {{ mgmt_ip_address }} diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh index a37f02af5e9..b3967ab21af 100755 --- a/ansible/testbed-cli.sh +++ b/ansible/testbed-cli.sh @@ -32,6 +32,7 @@ function usage echo " $0 [options] (create-master | destroy-master) " echo " $0 [options] restart-ptf " echo " $0 [options] set-l2 " + echo " $0 [options] deploy-l1 " echo " $0 [options] install-image " echo " $0 [options] install-dpu-image []" echo " $0 [options] collect-show-tech " @@ -95,6 +96,7 @@ function usage echo "To destroy Kubernetes master on a server: $0 -m k8s_ubuntu destroy-master 'k8s-server-name' ~/.password" echo "To restart ptf of specified testbed: $0 restart-ptf 'testbed-name' ~/.password" echo "To set DUT of specified testbed to l2 switch mode: $0 set-l2 'testbed-name' ~/.password" + echo "To deploy L1 (OCS) config for a testbed: $0 deploy-l1 'testbed-name' 'inventory' ~/.password" echo "To install an image on all DUTs in a testbed: $0 install-image 'testbed-name' 'inventory' 'image-url'" echo "To install an image on DPUs of a testbed: $0 install-dpu-image 'testbed-name' 'inventory' 'image-url' [dpu-index]" echo " Optional argument for install-dpu-image:" @@ -216,6 +218,18 @@ function converge_topo_if_needed backup_file="${topo_file}".bak if [[ "$use_converged_peers" == "True" ]]; then + # The converged (multi-VRF) peer model is implemented for cEOS + # neighbors only: a single cEOS VM hosts every merged sub-peer as a VRF, + # and only the cEOS startup-config templates render that VRF config. + # SONiC-VS / cisco / csonic neighbors have no converged render path, so + # reshaping the topology for them produces a DUT minigraph whose BGP + # neighbors the unconverged VS peers can't answer ("Not all bgp sessions + # established"). Gate on vm_type so non-cEOS deployments of a + # converged-enabled testbed behave exactly as they did historically. + if [[ "$vm_type" != "ceos" ]]; then + echo "use_converged_peers is true but vm_type='$vm_type' is not ceos; skipping converge (converged peer model is cEOS-only)." + return + fi echo "use_converged_peers is true, converging topo..." if [[ -f "$backup_file" ]];then @@ -308,8 +322,8 @@ function read_nut_file function start_vms { - if [[ $vm_type == ceos ]]; then - echo "VM type is ceos. No need to run start-vms. Please specify VM type using the -k option. Example: -k ceos" + if [[ $vm_type == ceos || $vm_type == csonic ]]; then + echo "VM type is $vm_type (container-based). No need to run start-vms." exit fi server=$1 @@ -324,8 +338,8 @@ function start_vms function stop_vms { - if [[ $vm_type == ceos ]]; then - echo "VM type is ceos. No need to run stop-vms. Please specify VM type using the -k option. Example: -k ceos" + if [[ $vm_type == ceos || $vm_type == csonic ]]; then + echo "VM type is $vm_type (container-based). No need to run stop-vms." exit fi server=$1 @@ -339,8 +353,8 @@ function stop_vms function start_topo_vms { - if [[ $vm_type == ceos ]]; then - echo "VM type is ceos. No need to run start-topo-vms. Please specify VM type using the -k option. Example: -k ceos" + if [[ $vm_type == ceos || $vm_type == csonic ]]; then + echo "VM type is $vm_type (container-based). No need to run start-topo-vms." exit fi testbed_name=$1 @@ -357,8 +371,8 @@ function start_topo_vms function stop_topo_vms { - if [[ $vm_type == ceos ]]; then - echo "VM type is ceos. No need to run stop-topo-vms. Please specify VM type using the -k option. Example: -k ceos" + if [[ $vm_type == ceos || $vm_type == csonic ]]; then + echo "VM type is $vm_type (container-based). No need to run stop-topo-vms." exit fi testbed_name=$1 @@ -670,6 +684,7 @@ function renumber_topo ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" \ -l "$server" -e testbed_name="$testbed_name" -e duts_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" \ -e topo="$topo" -e vm_set_name="$vm_set_name" -e ptf_imagename="$ptf_imagename" -e ptf_ipv6="$ptf_ipv6" \ + -e vm_type="$vm_type" \ -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" $@ @@ -732,7 +747,7 @@ function connect_vms read_file $1 - ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_connect_vms.yml --vault-password-file="$2" -l "$server" -e duts_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$vm_set_name" + ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_connect_vms.yml --vault-password-file="$2" -l "$server" -e duts_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$vm_set_name" -e vm_type="$vm_type" echo Done } @@ -743,7 +758,7 @@ function disconnect_vms read_file $1 - ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_disconnect_vms.yml --vault-password-file="$2" -l "$server" -e duts_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$vm_set_name" + ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_disconnect_vms.yml --vault-password-file="$2" -l "$server" -e duts_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$vm_set_name" -e vm_type="$vm_type" echo Done } @@ -923,7 +938,20 @@ function deploy_l1 echo "Devices to generate config for: $devices" echo "" - ansible-playbook -i "$inventory" deploy_config_on_testbed.yml --vault-password-file="$passfile" -l "$devices" -e testbed_name="$testbed_name" -e testbed_file=$tbfile -e deploy=true -e save=true -e config_duts=false -e reset_previous_connection=false$@ + # Toggle the OCS cross-connect CLI path. When true, reconcile + # cross-connects via the CLI and clear stale entries (the CLI persists + # automatically, no "config save" needed; l1_xconnect_clear_stale defaults + # to true). When false, fall back to the patch and reload path (gated by + # deploy=true) and reset previous connections. The CLI path and deploy=true + # are mutually exclusive. + use_l1_ocs_cli="${use_l1_ocs_cli:-true}" + if [[ "$use_l1_ocs_cli" == "true" ]]; then + ocs_options="-e use_l1_ocs_cli=true" + else + ocs_options="-e use_l1_ocs_cli=false -e deploy=true -e save=true -e reset_previous_connection=false" + fi + + ansible-playbook -i "$inventory" deploy_config_on_testbed.yml --vault-password-file="$passfile" -l "$devices" -e testbed_name="$testbed_name" -e testbed_file=$tbfile -e config_duts=false $ocs_options "$@" echo Done } diff --git a/ansible/testbed.yaml b/ansible/testbed.yaml index e22280e554b..7cd632c2036 100644 --- a/ansible/testbed.yaml +++ b/ansible/testbed.yaml @@ -283,3 +283,21 @@ inv_name: lab auto_recover: 'False' comment: BMC dual-mgmt testbed + +- conf-name: tor-dualtor3 + group-name: tor-dualtor + topo: dualtor-aa + ptf_image_name: docker-ptf + ptf: ptf_dtor3 + ptf_ip: 10.255.0.210/24 + ptf_ipv6: 2001:db8:1::20/64 + prober_type: hardware + server: server_1 + vm_base: VM0100 + dut: + - dtor3-dut0 + - dtor3-dut1 + inv_name: lab + auto_recover: 'True' + netns_mgmt_ip: 10.255.0.211/16 + comment: dualtor setup example diff --git a/ansible/testbed_renumber_vm_topology.yml b/ansible/testbed_renumber_vm_topology.yml index 57df21a3d24..56de9dc5096 100644 --- a/ansible/testbed_renumber_vm_topology.yml +++ b/ansible/testbed_renumber_vm_topology.yml @@ -83,6 +83,17 @@ set_fact: package_installation: false + # Defensive bootstrap: even though we set package_installation=false to skip + # heavy host setup (apt, sysctl, docker), we still need `uv` and the persistent + # venv at {{ sonic_testbed_venv }} because the renumber flow calls into + # control_mux_simulator.yml / control_nic_simulator.yml which `uv pip install` + # Flask/Werkzeug unconditionally. ensure_python_env.yml is idempotent: it + # checks for `uv` and the venv first and only installs what is missing. + - name: Ensure uv and Python venv exist on the host + include_role: + name: vm_set + tasks_from: ensure_python_env.yml + - name: Load topo variables include_vars: "vars/topo_{{ topo }}.yml" diff --git a/ansible/testbed_set_l2_mode.yml b/ansible/testbed_set_l2_mode.yml index aad141dcf72..b2f7e48f64b 100644 --- a/ansible/testbed_set_l2_mode.yml +++ b/ansible/testbed_set_l2_mode.yml @@ -37,8 +37,8 @@ dest: /etc/sonic/config_db.json become: yes - - name: Execute cli "config reload -y" to apply new config_db.json - shell: config reload -y + - name: Execute cli "config reload -y -f" to apply new config_db.json + shell: config reload -y -f become: yes - name: Wait for switch to become reachable again diff --git a/ansible/vars/topo_drh_tgen_route_conv.yml b/ansible/vars/topo_drh_tgen_route_conv.yml new file mode 100644 index 00000000000..7b2ebd7a714 --- /dev/null +++ b/ansible/vars/topo_drh_tgen_route_conv.yml @@ -0,0 +1,449 @@ +topology: + # dRH (Disaggregated Regional Hub) tgen route convergence topology + # 1 DUT (LowerRegionalHub) with 16 uplink ports to IXIA/tgen + # IXIA emulates UpperRegionalHub neighbors via PortChannels + # + # No downlink ports (FRH/LT2 not included in this topo) + + dut_num: 1 + VMs: + # Uplink ports - via Fanout to Ixia (T3/Spine emulation) + Snappi_URH_1: + vlans: + - 32 + vm_offset: 0 + Snappi_URH_2: + vlans: + - 33 + vm_offset: 1 + Snappi_URH_3: + vlans: + - 34 + vm_offset: 2 + Snappi_URH_4: + vlans: + - 35 + vm_offset: 3 + Snappi_URH_5: + vlans: + - 36 + vm_offset: 4 + Snappi_URH_6: + vlans: + - 37 + vm_offset: 5 + Snappi_URH_7: + vlans: + - 38 + vm_offset: 6 + Snappi_URH_8: + vlans: + - 39 + vm_offset: 7 + Snappi_URH_9: + vlans: + - 40 + vm_offset: 8 + Snappi_URH_10: + vlans: + - 41 + vm_offset: 9 + Snappi_URH_11: + vlans: + - 42 + vm_offset: 10 + Snappi_URH_12: + vlans: + - 43 + vm_offset: 11 + Snappi_URH_13: + vlans: + - 44 + vm_offset: 12 + Snappi_URH_14: + vlans: + - 45 + vm_offset: 13 + Snappi_URH_15: + vlans: + - 46 + vm_offset: 14 + Snappi_URH_16: + vlans: + - 47 + vm_offset: 15 + + DUT: + loopback: + ipv4: + - 10.1.0.1/32 + ipv6: + - FC00:10::1/128 + +configuration_properties: + common: + podset_number: 400 + tor_number: 16 + tor_subnet_number: 8 + max_tor_subnet_number: 32 + tor_subnet_size: 128 + dut_asn: 65100 + dut_type: LowerRegionalHub + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + core: + swrole: core + +configuration: + Snappi_URH_1: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.3.1.0 + - 2000:1:1:1::1 + interfaces: + Loopback0: + ipv4: 100.1.0.1/32 + ipv6: 2064:100::1/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.3.1.1/31 + ipv6: 2000:1:1:1::2/126 + bp_interface: + ipv4: 10.10.246.1/24 + ipv6: fc0a::2/64 + Snappi_URH_2: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.4.1.0 + - 2000:1:1:2::1 + interfaces: + Loopback0: + ipv4: 100.1.0.2/32 + ipv6: 2064:100::2/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.4.1.1/31 + ipv6: 2000:1:1:2::2/126 + bp_interface: + ipv4: 10.10.246.2/24 + ipv6: fc0a::3/64 + Snappi_URH_3: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.5.1.0 + - 2000:1:1:3::1 + interfaces: + Loopback0: + ipv4: 100.1.0.3/32 + ipv6: 2064:100::3/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.5.1.1/31 + ipv6: 2000:1:1:3::2/126 + bp_interface: + ipv4: 10.10.246.3/24 + ipv6: fc0a::4/64 + Snappi_URH_4: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.6.1.0 + - 2000:1:1:4::1 + interfaces: + Loopback0: + ipv4: 100.1.0.4/32 + ipv6: 2064:100::4/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.6.1.1/31 + ipv6: 2000:1:1:4::2/126 + bp_interface: + ipv4: 10.10.246.4/24 + ipv6: fc0a::5/64 + Snappi_URH_5: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.7.1.0 + - 2000:1:1:5::1 + interfaces: + Loopback0: + ipv4: 100.1.0.5/32 + ipv6: 2064:100::5/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.7.1.1/31 + ipv6: 2000:1:1:5::2/126 + bp_interface: + ipv4: 10.10.246.5/24 + ipv6: fc0a::6/64 + Snappi_URH_6: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.8.1.0 + - 2000:1:1:6::1 + interfaces: + Loopback0: + ipv4: 100.1.0.6/32 + ipv6: 2064:100::6/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.8.1.1/31 + ipv6: 2000:1:1:6::2/126 + bp_interface: + ipv4: 10.10.246.6/24 + ipv6: fc0a::7/64 + Snappi_URH_7: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.9.1.0 + - 2000:1:1:7::1 + interfaces: + Loopback0: + ipv4: 100.1.0.7/32 + ipv6: 2064:100::7/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.9.1.1/31 + ipv6: 2000:1:1:7::2/126 + bp_interface: + ipv4: 10.10.246.7/24 + ipv6: fc0a::8/64 + Snappi_URH_8: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.10.1.0 + - 2000:1:1:8::1 + interfaces: + Loopback0: + ipv4: 100.1.0.8/32 + ipv6: 2064:100::8/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.10.1.1/31 + ipv6: 2000:1:1:8::2/126 + bp_interface: + ipv4: 10.10.246.8/24 + ipv6: fc0a::9/64 + Snappi_URH_9: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.11.1.0 + - 2000:1:1:9::1 + interfaces: + Loopback0: + ipv4: 100.1.0.9/32 + ipv6: 2064:100::9/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.11.1.1/31 + ipv6: 2000:1:1:9::2/126 + bp_interface: + ipv4: 10.10.246.9/24 + ipv6: fc0a::a/64 + Snappi_URH_10: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.12.1.0 + - 2000:1:1:a::1 + interfaces: + Loopback0: + ipv4: 100.1.0.10/32 + ipv6: 2064:100::a/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.12.1.1/31 + ipv6: 2000:1:1:a::2/126 + bp_interface: + ipv4: 10.10.246.10/24 + ipv6: fc0a::b/64 + Snappi_URH_11: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.13.1.0 + - 2000:1:1:b::1 + interfaces: + Loopback0: + ipv4: 100.1.0.11/32 + ipv6: 2064:100::b/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.13.1.1/31 + ipv6: 2000:1:1:b::2/126 + bp_interface: + ipv4: 10.10.246.11/24 + ipv6: fc0a::c/64 + Snappi_URH_12: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.14.1.0 + - 2000:1:1:c::1 + interfaces: + Loopback0: + ipv4: 100.1.0.12/32 + ipv6: 2064:100::c/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.14.1.1/31 + ipv6: 2000:1:1:c::2/126 + bp_interface: + ipv4: 10.10.246.12/24 + ipv6: fc0a::d/64 + Snappi_URH_13: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.15.1.0 + - 2000:1:1:d::1 + interfaces: + Loopback0: + ipv4: 100.1.0.13/32 + ipv6: 2064:100::d/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.15.1.1/31 + ipv6: 2000:1:1:d::2/126 + bp_interface: + ipv4: 10.10.246.13/24 + ipv6: fc0a::e/64 + Snappi_URH_14: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.16.1.0 + - 2000:1:1:e::1 + interfaces: + Loopback0: + ipv4: 100.1.0.14/32 + ipv6: 2064:100::e/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.16.1.1/31 + ipv6: 2000:1:1:e::2/126 + bp_interface: + ipv4: 10.10.246.14/24 + ipv6: fc0a::f/64 + Snappi_URH_15: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.17.1.0 + - 2000:1:1:f::1 + interfaces: + Loopback0: + ipv4: 100.1.0.15/32 + ipv6: 2064:100::f/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.17.1.1/31 + ipv6: 2000:1:1:f::2/126 + bp_interface: + ipv4: 10.10.246.15/24 + ipv6: fc0a::10/64 + Snappi_URH_16: + properties: + - common + - core + bgp: + asn: 65400 + peers: + 65100: + - 20.18.1.0 + - 2000:1:1:10::1 + interfaces: + Loopback0: + ipv4: 100.1.0.16/32 + ipv6: 2064:100::10/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 20.18.1.1/31 + ipv6: 2000:1:1:10::2/126 + bp_interface: + ipv4: 10.10.246.16/24 + ipv6: fc0a::11/64 diff --git a/ansible/vars/topo_dualtor-aa-vpp.yml b/ansible/vars/topo_dualtor-aa-vpp.yml new file mode 120000 index 00000000000..6ee8cda187b --- /dev/null +++ b/ansible/vars/topo_dualtor-aa-vpp.yml @@ -0,0 +1 @@ +topo_dualtor-aa.yml \ No newline at end of file diff --git a/ansible/vars/topo_dualtor-vpp.yml b/ansible/vars/topo_dualtor-vpp.yml new file mode 120000 index 00000000000..efedefcf372 --- /dev/null +++ b/ansible/vars/topo_dualtor-vpp.yml @@ -0,0 +1 @@ +topo_dualtor.yml \ No newline at end of file diff --git a/ansible/vars/topo_t0-isolated-d32u32s2-mix.yml b/ansible/vars/topo_t0-isolated-d32u32s2-mix.yml index 2c793019bb1..f2502cc6b15 100644 --- a/ansible/vars/topo_t0-isolated-d32u32s2-mix.yml +++ b/ansible/vars/topo_t0-isolated-d32u32s2-mix.yml @@ -1,173 +1,173 @@ topology: host_interfaces: - 0 - - 15 - - 20 - - 35 - - 40 - - 55 + - 9 + - 12 + - 21 + - 24 + - 33 + - 36 + - 45 + - 48 + - 57 - 60 - - 75 - - 80 - - 95 - - 100 - - 115 + - 69 + - 72 + - 81 + - 84 + - 93 + - 96 + - 105 + - 108 + - 117 - 120 - - 135 - - 140 - - 155 - - 160 - - 175 + - 129 + - 132 + - 141 + - 144 + - 153 + - 156 + - 165 + - 168 + - 177 - 180 - - 195 - - 200 - - 215 - - 220 - - 235 - - 240 - - 255 - - 260 - - 275 - - 280 - - 295 - - 300 - - 315 + - 189 VMs: ARISTA01T1: vlans: - - 4 + - 2 vm_offset: 0 ARISTA02T1: vlans: - - 19 + - 11 vm_offset: 1 ARISTA03T1: vlans: - - 24 + - 14 vm_offset: 2 ARISTA04T1: vlans: - - 39 + - 23 vm_offset: 3 ARISTA05T1: vlans: - - 44 + - 26 vm_offset: 4 ARISTA06T1: vlans: - - 59 + - 35 vm_offset: 5 ARISTA07T1: vlans: - - 64 + - 38 vm_offset: 6 ARISTA08T1: vlans: - - 79 + - 47 vm_offset: 7 ARISTA09T1: vlans: - - 84 + - 50 vm_offset: 8 ARISTA10T1: vlans: - - 99 + - 59 vm_offset: 9 ARISTA11T1: vlans: - - 104 + - 62 vm_offset: 10 ARISTA12T1: vlans: - - 119 + - 71 vm_offset: 11 ARISTA13T1: vlans: - - 124 + - 74 vm_offset: 12 ARISTA14T1: vlans: - - 139 + - 83 vm_offset: 13 ARISTA15T1: vlans: - - 144 + - 86 vm_offset: 14 ARISTA16T1: vlans: - - 159 + - 95 vm_offset: 15 ARISTA17T1: vlans: - - 164 + - 98 vm_offset: 16 ARISTA18T1: vlans: - - 179 + - 107 vm_offset: 17 ARISTA19T1: vlans: - - 184 + - 110 vm_offset: 18 ARISTA20T1: vlans: - - 199 + - 119 vm_offset: 19 ARISTA21T1: vlans: - - 204 + - 122 vm_offset: 20 ARISTA22T1: vlans: - - 219 + - 131 vm_offset: 21 ARISTA23T1: vlans: - - 224 + - 134 vm_offset: 22 ARISTA24T1: vlans: - - 239 + - 143 vm_offset: 23 ARISTA25T1: vlans: - - 244 + - 146 vm_offset: 24 ARISTA26T1: vlans: - - 259 + - 155 vm_offset: 25 ARISTA27T1: vlans: - - 264 + - 158 vm_offset: 26 ARISTA28T1: vlans: - - 279 + - 167 vm_offset: 27 ARISTA29T1: vlans: - - 284 + - 170 vm_offset: 28 ARISTA30T1: vlans: - - 299 + - 179 vm_offset: 29 ARISTA31T1: vlans: - - 304 + - 182 vm_offset: 30 ARISTA32T1: vlans: - - 319 + - 191 vm_offset: 31 ARISTA01PT0: vlans: - - 320 + - 192 vm_offset: 32 ARISTA02PT0: vlans: - - 321 + - 193 vm_offset: 33 DUT: vlan_configs: @@ -175,45 +175,45 @@ topology: one_vlan_a: Vlan1000: id: 1000 - intfs: [0, 15, 20, 35, 40, 55, 60, 75, 80, 95, 100, 115, 120, 135, 140, 155, 160, 175, 180, 195, 200, 215, 220, 235, 240, 255, 260, 275, 280, 295, 300, 315] + intfs: [0, 9, 12, 21, 24, 33, 36, 45, 48, 57, 60, 69, 72, 81, 84, 93, 96, 105, 108, 117, 120, 129, 132, 141, 144, 153, 156, 165, 168, 177, 180, 189] prefix: 192.168.0.1/21 prefix_v6: fc02:1000::1/64 tag: 1000 two_vlan_a: Vlan1000: id: 1000 - intfs: [0, 15, 20, 35, 40, 55, 60, 75, 80, 95, 100, 115, 120, 135, 140, 155] + intfs: [0, 9, 12, 21, 24, 33, 36, 45, 48, 57, 60, 69, 72, 81, 84, 93] prefix: 192.168.0.1/22 prefix_v6: fc02:100::1/64 tag: 1000 Vlan1100: id: 1100 - intfs: [160, 175, 180, 195, 200, 215, 220, 235, 240, 255, 260, 275, 280, 295, 300, 315] + intfs: [96, 105, 108, 117, 120, 129, 132, 141, 144, 153, 156, 165, 168, 177, 180, 189] prefix: 192.168.4.1/22 prefix_v6: fc02:101::1/64 tag: 1100 four_vlan_a: Vlan1000: id: 1000 - intfs: [0, 15, 20, 35, 40, 55, 60, 75] + intfs: [0, 9, 12, 21, 24, 33, 36, 45] prefix: 192.168.0.1/22 prefix_v6: fc02:100::1/64 tag: 1000 Vlan1100: id: 1100 - intfs: [80, 95, 100, 115, 120, 135, 140, 155] + intfs: [48, 57, 60, 69, 72, 81, 84, 93] prefix: 192.168.4.1/22 prefix_v6: fc02:101::1/64 tag: 1100 Vlan1200: id: 1200 - intfs: [160, 175, 180, 195, 200, 215, 220, 235] + intfs: [96, 105, 108, 117, 120, 129, 132, 141] prefix: 192.168.8.1/22 prefix_v6: fc02:102::1/64 tag: 1200 Vlan1300: id: 1300 - intfs: [240, 255, 260, 275, 280, 295, 300, 315] + intfs: [144, 153, 156, 165, 168, 177, 180, 189] prefix: 192.168.12.1/22 prefix_v6: fc02:103::1/64 tag: 1300 @@ -247,18 +247,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.8 - - fc00::11 + - 10.0.0.4 + - fc00::9 interfaces: Loopback0: - ipv4: 100.1.0.5/32 - ipv6: 2064:100:0:5::/128 + ipv4: 100.1.0.3/32 + ipv6: 2064:100:0:3::/128 Ethernet1: - ipv4: 10.0.0.9/31 - ipv6: fc00::12/126 + ipv4: 10.0.0.5/31 + ipv6: fc00::a/126 bp_interface: - ipv4: 10.10.246.6/22 - ipv6: fc0a::6/64 + ipv4: 10.10.246.4/22 + ipv6: fc0a::4/64 ARISTA02T1: properties: - common @@ -267,18 +267,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.38 - - fc00::4d + - 10.0.0.22 + - fc00::2d interfaces: Loopback0: - ipv4: 100.1.0.20/32 - ipv6: 2064:100:0:14::/128 + ipv4: 100.1.0.12/32 + ipv6: 2064:100:0:c::/128 Ethernet1: - ipv4: 10.0.0.39/31 - ipv6: fc00::4e/126 + ipv4: 10.0.0.23/31 + ipv6: fc00::2e/126 bp_interface: - ipv4: 10.10.246.21/22 - ipv6: fc0a::15/64 + ipv4: 10.10.246.13/22 + ipv6: fc0a::d/64 ARISTA03T1: properties: - common @@ -287,18 +287,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.48 - - fc00::61 + - 10.0.0.28 + - fc00::39 interfaces: Loopback0: - ipv4: 100.1.0.25/32 - ipv6: 2064:100:0:19::/128 + ipv4: 100.1.0.15/32 + ipv6: 2064:100:0:f::/128 Ethernet1: - ipv4: 10.0.0.49/31 - ipv6: fc00::62/126 + ipv4: 10.0.0.29/31 + ipv6: fc00::3a/126 bp_interface: - ipv4: 10.10.246.26/22 - ipv6: fc0a::1a/64 + ipv4: 10.10.246.16/22 + ipv6: fc0a::10/64 ARISTA04T1: properties: - common @@ -307,18 +307,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.78 - - fc00::9d + - 10.0.0.46 + - fc00::5d interfaces: Loopback0: - ipv4: 100.1.0.40/32 - ipv6: 2064:100:0:28::/128 + ipv4: 100.1.0.24/32 + ipv6: 2064:100:0:18::/128 Ethernet1: - ipv4: 10.0.0.79/31 - ipv6: fc00::9e/126 + ipv4: 10.0.0.47/31 + ipv6: fc00::5e/126 bp_interface: - ipv4: 10.10.246.41/22 - ipv6: fc0a::29/64 + ipv4: 10.10.246.25/22 + ipv6: fc0a::19/64 ARISTA05T1: properties: - common @@ -327,18 +327,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.88 - - fc00::b1 + - 10.0.0.52 + - fc00::69 interfaces: Loopback0: - ipv4: 100.1.0.45/32 - ipv6: 2064:100:0:2d::/128 + ipv4: 100.1.0.27/32 + ipv6: 2064:100:0:1b::/128 Ethernet1: - ipv4: 10.0.0.89/31 - ipv6: fc00::b2/126 + ipv4: 10.0.0.53/31 + ipv6: fc00::6a/126 bp_interface: - ipv4: 10.10.246.46/22 - ipv6: fc0a::2e/64 + ipv4: 10.10.246.28/22 + ipv6: fc0a::1c/64 ARISTA06T1: properties: - common @@ -347,18 +347,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.118 - - fc00::ed + - 10.0.0.70 + - fc00::8d interfaces: Loopback0: - ipv4: 100.1.0.60/32 - ipv6: 2064:100:0:3c::/128 + ipv4: 100.1.0.36/32 + ipv6: 2064:100:0:24::/128 Ethernet1: - ipv4: 10.0.0.119/31 - ipv6: fc00::ee/126 + ipv4: 10.0.0.71/31 + ipv6: fc00::8e/126 bp_interface: - ipv4: 10.10.246.61/22 - ipv6: fc0a::3d/64 + ipv4: 10.10.246.37/22 + ipv6: fc0a::25/64 ARISTA07T1: properties: - common @@ -367,18 +367,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.128 - - fc00::101 + - 10.0.0.76 + - fc00::99 interfaces: Loopback0: - ipv4: 100.1.0.65/32 - ipv6: 2064:100:0:41::/128 + ipv4: 100.1.0.39/32 + ipv6: 2064:100:0:27::/128 Ethernet1: - ipv4: 10.0.0.129/31 - ipv6: fc00::102/126 + ipv4: 10.0.0.77/31 + ipv6: fc00::9a/126 bp_interface: - ipv4: 10.10.246.66/22 - ipv6: fc0a::42/64 + ipv4: 10.10.246.40/22 + ipv6: fc0a::28/64 ARISTA08T1: properties: - common @@ -387,18 +387,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.158 - - fc00::13d + - 10.0.0.94 + - fc00::bd interfaces: Loopback0: - ipv4: 100.1.0.80/32 - ipv6: 2064:100:0:50::/128 + ipv4: 100.1.0.48/32 + ipv6: 2064:100:0:30::/128 Ethernet1: - ipv4: 10.0.0.159/31 - ipv6: fc00::13e/126 + ipv4: 10.0.0.95/31 + ipv6: fc00::be/126 bp_interface: - ipv4: 10.10.246.81/22 - ipv6: fc0a::51/64 + ipv4: 10.10.246.49/22 + ipv6: fc0a::31/64 ARISTA09T1: properties: - common @@ -407,18 +407,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.168 - - fc00::151 + - 10.0.0.100 + - fc00::c9 interfaces: Loopback0: - ipv4: 100.1.0.85/32 - ipv6: 2064:100:0:55::/128 + ipv4: 100.1.0.51/32 + ipv6: 2064:100:0:33::/128 Ethernet1: - ipv4: 10.0.0.169/31 - ipv6: fc00::152/126 + ipv4: 10.0.0.101/31 + ipv6: fc00::ca/126 bp_interface: - ipv4: 10.10.246.86/22 - ipv6: fc0a::56/64 + ipv4: 10.10.246.52/22 + ipv6: fc0a::34/64 ARISTA10T1: properties: - common @@ -427,18 +427,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.198 - - fc00::18d + - 10.0.0.118 + - fc00::ed interfaces: Loopback0: - ipv4: 100.1.0.100/32 - ipv6: 2064:100:0:64::/128 + ipv4: 100.1.0.60/32 + ipv6: 2064:100:0:3c::/128 Ethernet1: - ipv4: 10.0.0.199/31 - ipv6: fc00::18e/126 + ipv4: 10.0.0.119/31 + ipv6: fc00::ee/126 bp_interface: - ipv4: 10.10.246.101/22 - ipv6: fc0a::65/64 + ipv4: 10.10.246.61/22 + ipv6: fc0a::3d/64 ARISTA11T1: properties: - common @@ -447,18 +447,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.208 - - fc00::1a1 + - 10.0.0.124 + - fc00::f9 interfaces: Loopback0: - ipv4: 100.1.0.105/32 - ipv6: 2064:100:0:69::/128 + ipv4: 100.1.0.63/32 + ipv6: 2064:100:0:3f::/128 Ethernet1: - ipv4: 10.0.0.209/31 - ipv6: fc00::1a2/126 + ipv4: 10.0.0.125/31 + ipv6: fc00::fa/126 bp_interface: - ipv4: 10.10.246.106/22 - ipv6: fc0a::6a/64 + ipv4: 10.10.246.64/22 + ipv6: fc0a::40/64 ARISTA12T1: properties: - common @@ -467,18 +467,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.238 - - fc00::1dd + - 10.0.0.142 + - fc00::11d interfaces: Loopback0: - ipv4: 100.1.0.120/32 - ipv6: 2064:100:0:78::/128 + ipv4: 100.1.0.72/32 + ipv6: 2064:100:0:48::/128 Ethernet1: - ipv4: 10.0.0.239/31 - ipv6: fc00::1de/126 + ipv4: 10.0.0.143/31 + ipv6: fc00::11e/126 bp_interface: - ipv4: 10.10.246.121/22 - ipv6: fc0a::79/64 + ipv4: 10.10.246.73/22 + ipv6: fc0a::49/64 ARISTA13T1: properties: - common @@ -487,18 +487,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.0.248 - - fc00::1f1 + - 10.0.0.148 + - fc00::129 interfaces: Loopback0: - ipv4: 100.1.0.125/32 - ipv6: 2064:100:0:7d::/128 + ipv4: 100.1.0.75/32 + ipv6: 2064:100:0:4b::/128 Ethernet1: - ipv4: 10.0.0.249/31 - ipv6: fc00::1f2/126 + ipv4: 10.0.0.149/31 + ipv6: fc00::12a/126 bp_interface: - ipv4: 10.10.246.126/22 - ipv6: fc0a::7e/64 + ipv4: 10.10.246.76/22 + ipv6: fc0a::4c/64 ARISTA14T1: properties: - common @@ -507,18 +507,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.22 - - fc00::22d + - 10.0.0.166 + - fc00::14d interfaces: Loopback0: - ipv4: 100.1.0.140/32 - ipv6: 2064:100:0:8c::/128 + ipv4: 100.1.0.84/32 + ipv6: 2064:100:0:54::/128 Ethernet1: - ipv4: 10.0.1.23/31 - ipv6: fc00::22e/126 + ipv4: 10.0.0.167/31 + ipv6: fc00::14e/126 bp_interface: - ipv4: 10.10.246.141/22 - ipv6: fc0a::8d/64 + ipv4: 10.10.246.85/22 + ipv6: fc0a::55/64 ARISTA15T1: properties: - common @@ -527,18 +527,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.32 - - fc00::241 + - 10.0.0.172 + - fc00::159 interfaces: Loopback0: - ipv4: 100.1.0.145/32 - ipv6: 2064:100:0:91::/128 + ipv4: 100.1.0.87/32 + ipv6: 2064:100:0:57::/128 Ethernet1: - ipv4: 10.0.1.33/31 - ipv6: fc00::242/126 + ipv4: 10.0.0.173/31 + ipv6: fc00::15a/126 bp_interface: - ipv4: 10.10.246.146/22 - ipv6: fc0a::92/64 + ipv4: 10.10.246.88/22 + ipv6: fc0a::58/64 ARISTA16T1: properties: - common @@ -547,18 +547,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.62 - - fc00::27d + - 10.0.0.190 + - fc00::17d interfaces: Loopback0: - ipv4: 100.1.0.160/32 - ipv6: 2064:100:0:a0::/128 + ipv4: 100.1.0.96/32 + ipv6: 2064:100:0:60::/128 Ethernet1: - ipv4: 10.0.1.63/31 - ipv6: fc00::27e/126 + ipv4: 10.0.0.191/31 + ipv6: fc00::17e/126 bp_interface: - ipv4: 10.10.246.161/22 - ipv6: fc0a::a1/64 + ipv4: 10.10.246.97/22 + ipv6: fc0a::61/64 ARISTA17T1: properties: - common @@ -567,18 +567,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.72 - - fc00::291 + - 10.0.0.196 + - fc00::189 interfaces: Loopback0: - ipv4: 100.1.0.165/32 - ipv6: 2064:100:0:a5::/128 + ipv4: 100.1.0.99/32 + ipv6: 2064:100:0:63::/128 Ethernet1: - ipv4: 10.0.1.73/31 - ipv6: fc00::292/126 + ipv4: 10.0.0.197/31 + ipv6: fc00::18a/126 bp_interface: - ipv4: 10.10.246.166/22 - ipv6: fc0a::a6/64 + ipv4: 10.10.246.100/22 + ipv6: fc0a::64/64 ARISTA18T1: properties: - common @@ -587,18 +587,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.102 - - fc00::2cd + - 10.0.0.214 + - fc00::1ad interfaces: Loopback0: - ipv4: 100.1.0.180/32 - ipv6: 2064:100:0:b4::/128 + ipv4: 100.1.0.108/32 + ipv6: 2064:100:0:6c::/128 Ethernet1: - ipv4: 10.0.1.103/31 - ipv6: fc00::2ce/126 + ipv4: 10.0.0.215/31 + ipv6: fc00::1ae/126 bp_interface: - ipv4: 10.10.246.181/22 - ipv6: fc0a::b5/64 + ipv4: 10.10.246.109/22 + ipv6: fc0a::6d/64 ARISTA19T1: properties: - common @@ -607,18 +607,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.112 - - fc00::2e1 + - 10.0.0.220 + - fc00::1b9 interfaces: Loopback0: - ipv4: 100.1.0.185/32 - ipv6: 2064:100:0:b9::/128 + ipv4: 100.1.0.111/32 + ipv6: 2064:100:0:6f::/128 Ethernet1: - ipv4: 10.0.1.113/31 - ipv6: fc00::2e2/126 + ipv4: 10.0.0.221/31 + ipv6: fc00::1ba/126 bp_interface: - ipv4: 10.10.246.186/22 - ipv6: fc0a::ba/64 + ipv4: 10.10.246.112/22 + ipv6: fc0a::70/64 ARISTA20T1: properties: - common @@ -627,18 +627,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.142 - - fc00::31d + - 10.0.0.238 + - fc00::1dd interfaces: Loopback0: - ipv4: 100.1.0.200/32 - ipv6: 2064:100:0:c8::/128 + ipv4: 100.1.0.120/32 + ipv6: 2064:100:0:78::/128 Ethernet1: - ipv4: 10.0.1.143/31 - ipv6: fc00::31e/126 + ipv4: 10.0.0.239/31 + ipv6: fc00::1de/126 bp_interface: - ipv4: 10.10.246.201/22 - ipv6: fc0a::c9/64 + ipv4: 10.10.246.121/22 + ipv6: fc0a::79/64 ARISTA21T1: properties: - common @@ -647,18 +647,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.152 - - fc00::331 + - 10.0.0.244 + - fc00::1e9 interfaces: Loopback0: - ipv4: 100.1.0.205/32 - ipv6: 2064:100:0:cd::/128 + ipv4: 100.1.0.123/32 + ipv6: 2064:100:0:7b::/128 Ethernet1: - ipv4: 10.0.1.153/31 - ipv6: fc00::332/126 + ipv4: 10.0.0.245/31 + ipv6: fc00::1ea/126 bp_interface: - ipv4: 10.10.246.206/22 - ipv6: fc0a::ce/64 + ipv4: 10.10.246.124/22 + ipv6: fc0a::7c/64 ARISTA22T1: properties: - common @@ -667,18 +667,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.182 - - fc00::36d + - 10.0.1.6 + - fc00::20d interfaces: Loopback0: - ipv4: 100.1.0.220/32 - ipv6: 2064:100:0:dc::/128 + ipv4: 100.1.0.132/32 + ipv6: 2064:100:0:84::/128 Ethernet1: - ipv4: 10.0.1.183/31 - ipv6: fc00::36e/126 + ipv4: 10.0.1.7/31 + ipv6: fc00::20e/126 bp_interface: - ipv4: 10.10.246.221/22 - ipv6: fc0a::dd/64 + ipv4: 10.10.246.133/22 + ipv6: fc0a::85/64 ARISTA23T1: properties: - common @@ -687,18 +687,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.192 - - fc00::381 + - 10.0.1.12 + - fc00::219 interfaces: Loopback0: - ipv4: 100.1.0.225/32 - ipv6: 2064:100:0:e1::/128 + ipv4: 100.1.0.135/32 + ipv6: 2064:100:0:87::/128 Ethernet1: - ipv4: 10.0.1.193/31 - ipv6: fc00::382/126 + ipv4: 10.0.1.13/31 + ipv6: fc00::21a/126 bp_interface: - ipv4: 10.10.246.226/22 - ipv6: fc0a::e2/64 + ipv4: 10.10.246.136/22 + ipv6: fc0a::88/64 ARISTA24T1: properties: - common @@ -707,18 +707,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.222 - - fc00::3bd + - 10.0.1.30 + - fc00::23d interfaces: Loopback0: - ipv4: 100.1.0.240/32 - ipv6: 2064:100:0:f0::/128 + ipv4: 100.1.0.144/32 + ipv6: 2064:100:0:90::/128 Ethernet1: - ipv4: 10.0.1.223/31 - ipv6: fc00::3be/126 + ipv4: 10.0.1.31/31 + ipv6: fc00::23e/126 bp_interface: - ipv4: 10.10.246.241/22 - ipv6: fc0a::f1/64 + ipv4: 10.10.246.145/22 + ipv6: fc0a::91/64 ARISTA25T1: properties: - common @@ -727,18 +727,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.1.232 - - fc00::3d1 + - 10.0.1.36 + - fc00::249 interfaces: Loopback0: - ipv4: 100.1.0.245/32 - ipv6: 2064:100:0:f5::/128 + ipv4: 100.1.0.147/32 + ipv6: 2064:100:0:93::/128 Ethernet1: - ipv4: 10.0.1.233/31 - ipv6: fc00::3d2/126 + ipv4: 10.0.1.37/31 + ipv6: fc00::24a/126 bp_interface: - ipv4: 10.10.246.246/22 - ipv6: fc0a::f6/64 + ipv4: 10.10.246.148/22 + ipv6: fc0a::94/64 ARISTA26T1: properties: - common @@ -747,18 +747,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.6 - - fc00::40d + - 10.0.1.54 + - fc00::26d interfaces: Loopback0: - ipv4: 100.1.1.4/32 - ipv6: 2064:100:0:104::/128 + ipv4: 100.1.0.156/32 + ipv6: 2064:100:0:9c::/128 Ethernet1: - ipv4: 10.0.2.7/31 - ipv6: fc00::40e/126 + ipv4: 10.0.1.55/31 + ipv6: fc00::26e/126 bp_interface: - ipv4: 10.10.247.5/22 - ipv6: fc0a::105/64 + ipv4: 10.10.246.157/22 + ipv6: fc0a::9d/64 ARISTA27T1: properties: - common @@ -767,18 +767,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.16 - - fc00::421 + - 10.0.1.60 + - fc00::279 interfaces: Loopback0: - ipv4: 100.1.1.9/32 - ipv6: 2064:100:0:109::/128 + ipv4: 100.1.0.159/32 + ipv6: 2064:100:0:9f::/128 Ethernet1: - ipv4: 10.0.2.17/31 - ipv6: fc00::422/126 + ipv4: 10.0.1.61/31 + ipv6: fc00::27a/126 bp_interface: - ipv4: 10.10.247.10/22 - ipv6: fc0a::10a/64 + ipv4: 10.10.246.160/22 + ipv6: fc0a::a0/64 ARISTA28T1: properties: - common @@ -787,18 +787,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.46 - - fc00::45d + - 10.0.1.78 + - fc00::29d interfaces: Loopback0: - ipv4: 100.1.1.24/32 - ipv6: 2064:100:0:118::/128 + ipv4: 100.1.0.168/32 + ipv6: 2064:100:0:a8::/128 Ethernet1: - ipv4: 10.0.2.47/31 - ipv6: fc00::45e/126 + ipv4: 10.0.1.79/31 + ipv6: fc00::29e/126 bp_interface: - ipv4: 10.10.247.25/22 - ipv6: fc0a::119/64 + ipv4: 10.10.246.169/22 + ipv6: fc0a::a9/64 ARISTA29T1: properties: - common @@ -807,18 +807,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.56 - - fc00::471 + - 10.0.1.84 + - fc00::2a9 interfaces: Loopback0: - ipv4: 100.1.1.29/32 - ipv6: 2064:100:0:11d::/128 + ipv4: 100.1.0.171/32 + ipv6: 2064:100:0:ab::/128 Ethernet1: - ipv4: 10.0.2.57/31 - ipv6: fc00::472/126 + ipv4: 10.0.1.85/31 + ipv6: fc00::2aa/126 bp_interface: - ipv4: 10.10.247.30/22 - ipv6: fc0a::11e/64 + ipv4: 10.10.246.172/22 + ipv6: fc0a::ac/64 ARISTA30T1: properties: - common @@ -827,18 +827,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.86 - - fc00::4ad + - 10.0.1.102 + - fc00::2cd interfaces: Loopback0: - ipv4: 100.1.1.44/32 - ipv6: 2064:100:0:12c::/128 + ipv4: 100.1.0.180/32 + ipv6: 2064:100:0:b4::/128 Ethernet1: - ipv4: 10.0.2.87/31 - ipv6: fc00::4ae/126 + ipv4: 10.0.1.103/31 + ipv6: fc00::2ce/126 bp_interface: - ipv4: 10.10.247.45/22 - ipv6: fc0a::12d/64 + ipv4: 10.10.246.181/22 + ipv6: fc0a::b5/64 ARISTA31T1: properties: - common @@ -847,18 +847,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.96 - - fc00::4c1 + - 10.0.1.108 + - fc00::2d9 interfaces: Loopback0: - ipv4: 100.1.1.49/32 - ipv6: 2064:100:0:131::/128 + ipv4: 100.1.0.183/32 + ipv6: 2064:100:0:b7::/128 Ethernet1: - ipv4: 10.0.2.97/31 - ipv6: fc00::4c2/126 + ipv4: 10.0.1.109/31 + ipv6: fc00::2da/126 bp_interface: - ipv4: 10.10.247.50/22 - ipv6: fc0a::132/64 + ipv4: 10.10.246.184/22 + ipv6: fc0a::b8/64 ARISTA32T1: properties: - common @@ -867,18 +867,18 @@ configuration: asn: 64600 peers: 65100: - - 10.0.2.126 - - fc00::4fd + - 10.0.1.126 + - fc00::2fd interfaces: Loopback0: - ipv4: 100.1.1.64/32 - ipv6: 2064:100:0:140::/128 + ipv4: 100.1.0.192/32 + ipv6: 2064:100:0:c0::/128 Ethernet1: - ipv4: 10.0.2.127/31 - ipv6: fc00::4fe/126 + ipv4: 10.0.1.127/31 + ipv6: fc00::2fe/126 bp_interface: - ipv4: 10.10.247.65/22 - ipv6: fc0a::141/64 + ipv4: 10.10.246.193/22 + ipv6: fc0a::c1/64 ARISTA01PT0: properties: - common @@ -887,18 +887,18 @@ configuration: asn: 65101 peers: 65100: - - 10.0.2.128 - - fc00::501 + - 10.0.1.128 + - fc00::301 interfaces: Loopback0: - ipv4: 100.1.1.65/32 - ipv6: 2064:100:0:141::/128 + ipv4: 100.1.0.193/32 + ipv6: 2064:100:0:c1::/128 Ethernet1: - ipv4: 10.0.2.129/31 - ipv6: fc00::502/126 + ipv4: 10.0.1.129/31 + ipv6: fc00::302/126 bp_interface: - ipv4: 10.10.247.66/22 - ipv6: fc0a::142/64 + ipv4: 10.10.246.194/22 + ipv6: fc0a::c2/64 ARISTA02PT0: properties: - common @@ -907,15 +907,15 @@ configuration: asn: 65102 peers: 65100: - - 10.0.2.130 - - fc00::505 + - 10.0.1.130 + - fc00::305 interfaces: Loopback0: - ipv4: 100.1.1.66/32 - ipv6: 2064:100:0:142::/128 + ipv4: 100.1.0.194/32 + ipv6: 2064:100:0:c2::/128 Ethernet1: - ipv4: 10.0.2.131/31 - ipv6: fc00::506/126 + ipv4: 10.0.1.131/31 + ipv6: fc00::306/126 bp_interface: - ipv4: 10.10.247.67/22 - ipv6: fc0a::143/64 + ipv4: 10.10.246.195/22 + ipv6: fc0a::c3/64 diff --git a/ansible/vars/topo_t1-smartswitch-ha.yml b/ansible/vars/topo_t1-smartswitch-ha.yml index b2ab89639f3..b8ba2a7a575 100644 --- a/ansible/vars/topo_t1-smartswitch-ha.yml +++ b/ansible/vars/topo_t1-smartswitch-ha.yml @@ -1,4 +1,5 @@ topology: + topo_type: t1-smartswitch-ha dut_num: 2 VMs: ARISTA01T2: diff --git a/ansible/vars/topo_t2_single_node_max_64p.yml b/ansible/vars/topo_t2_single_node_max_64p.yml index 491e0ccd874..4dae8bf1e3d 100644 --- a/ansible/vars/topo_t2_single_node_max_64p.yml +++ b/ansible/vars/topo_t2_single_node_max_64p.yml @@ -334,7 +334,9 @@ configuration_properties: tor_subnet_number: 8 max_tor_subnet_number: 32 tor_subnet_size: 128 - dut_asn: 65100 + dut_asn: 66000 + dut_confed_asn: 65100 + dut_confed_peers: 65300 dut_type: UpperSpineRouter nhipv4: 10.10.246.254 nhipv6: fc0a::ff @@ -349,6 +351,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -372,6 +375,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -395,6 +399,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -418,6 +423,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -441,6 +447,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -464,6 +471,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -487,6 +495,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -510,6 +519,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -533,6 +543,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -556,6 +567,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -579,6 +591,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -602,6 +615,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -625,6 +639,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -648,6 +663,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -671,6 +687,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -694,6 +711,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -717,6 +735,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -740,6 +759,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -763,6 +783,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -786,6 +807,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -809,6 +831,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -832,6 +855,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -855,6 +879,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -878,6 +903,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -901,6 +927,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -924,6 +951,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -947,6 +975,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -970,6 +999,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -993,6 +1023,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1016,6 +1047,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1039,6 +1071,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1062,6 +1095,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1085,9 +1119,11 @@ configuration: - common - leaf bgp: - asn: 64600 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.128 - fc00::101 interfaces: @@ -1106,9 +1142,11 @@ configuration: - common - leaf bgp: - asn: 64610 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.130 - fc00::105 interfaces: @@ -1127,9 +1165,11 @@ configuration: - common - leaf bgp: - asn: 64620 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.132 - fc00::109 interfaces: @@ -1148,9 +1188,11 @@ configuration: - common - leaf bgp: - asn: 64630 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.134 - fc00::10d interfaces: @@ -1169,9 +1211,11 @@ configuration: - common - leaf bgp: - asn: 64640 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.136 - fc00::111 interfaces: @@ -1190,9 +1234,11 @@ configuration: - common - leaf bgp: - asn: 64650 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.138 - fc00::115 interfaces: @@ -1211,9 +1257,11 @@ configuration: - common - leaf bgp: - asn: 64660 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.140 - fc00::119 interfaces: @@ -1232,9 +1280,11 @@ configuration: - common - leaf bgp: - asn: 64670 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.142 - fc00::11d interfaces: @@ -1253,9 +1303,11 @@ configuration: - common - leaf bgp: - asn: 64680 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.144 - fc00::121 interfaces: @@ -1274,9 +1326,11 @@ configuration: - common - leaf bgp: - asn: 64690 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.146 - fc00::125 interfaces: @@ -1295,9 +1349,11 @@ configuration: - common - leaf bgp: - asn: 64700 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.148 - fc00::129 interfaces: @@ -1316,9 +1372,11 @@ configuration: - common - leaf bgp: - asn: 64710 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.150 - fc00::12d interfaces: @@ -1337,9 +1395,11 @@ configuration: - common - leaf bgp: - asn: 64720 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.152 - fc00::131 interfaces: @@ -1358,9 +1418,11 @@ configuration: - common - leaf bgp: - asn: 64730 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.154 - fc00::135 interfaces: @@ -1379,9 +1441,11 @@ configuration: - common - leaf bgp: - asn: 64740 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.156 - fc00::139 interfaces: @@ -1400,9 +1464,11 @@ configuration: - common - leaf bgp: - asn: 64750 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.158 - fc00::13d interfaces: @@ -1421,9 +1487,11 @@ configuration: - common - leaf bgp: - asn: 64760 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.160 - fc00::141 interfaces: @@ -1442,9 +1510,11 @@ configuration: - common - leaf bgp: - asn: 64770 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.162 - fc00::145 interfaces: @@ -1463,9 +1533,11 @@ configuration: - common - leaf bgp: - asn: 64780 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.164 - fc00::149 interfaces: @@ -1484,9 +1556,11 @@ configuration: - common - leaf bgp: - asn: 64790 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.166 - fc00::14d interfaces: @@ -1505,9 +1579,11 @@ configuration: - common - leaf bgp: - asn: 64800 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.168 - fc00::151 interfaces: @@ -1526,9 +1602,11 @@ configuration: - common - leaf bgp: - asn: 64810 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.170 - fc00::155 interfaces: @@ -1547,9 +1625,11 @@ configuration: - common - leaf bgp: - asn: 64820 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.172 - fc00::159 interfaces: @@ -1568,9 +1648,11 @@ configuration: - common - leaf bgp: - asn: 64830 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.174 - fc00::15d interfaces: @@ -1589,9 +1671,11 @@ configuration: - common - leaf bgp: - asn: 64840 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.176 - fc00::161 interfaces: @@ -1610,9 +1694,11 @@ configuration: - common - leaf bgp: - asn: 64850 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.178 - fc00::165 interfaces: @@ -1631,9 +1717,11 @@ configuration: - common - leaf bgp: - asn: 64860 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.180 - fc00::169 interfaces: @@ -1652,9 +1740,11 @@ configuration: - common - leaf bgp: - asn: 64870 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.182 - fc00::16d interfaces: @@ -1673,9 +1763,11 @@ configuration: - common - leaf bgp: - asn: 64880 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.184 - fc00::171 interfaces: @@ -1694,9 +1786,11 @@ configuration: - common - leaf bgp: - asn: 64890 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.186 - fc00::175 interfaces: @@ -1715,9 +1809,11 @@ configuration: - common - leaf bgp: - asn: 64900 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.188 - fc00::179 interfaces: @@ -1736,9 +1832,11 @@ configuration: - common - leaf bgp: - asn: 64910 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.190 - fc00::17d interfaces: diff --git a/ansible/vars/topo_t2_single_node_max_64p_v2.yml b/ansible/vars/topo_t2_single_node_max_64p_v2.yml index 1af8edb0b7c..2e36d41c23f 100644 --- a/ansible/vars/topo_t2_single_node_max_64p_v2.yml +++ b/ansible/vars/topo_t2_single_node_max_64p_v2.yml @@ -329,13 +329,15 @@ topology: configuration_properties: common: - dut_asn: 65100 - dut_type: UpperSpineRouter podset_number: 400 tor_number: 16 tor_subnet_number: 8 max_tor_subnet_number: 32 tor_subnet_size: 128 + dut_asn: 66000 + dut_confed_asn: 65100 + dut_confed_peers: 65300 + dut_type: UpperSpineRouter nhipv4: 10.10.246.254 nhipv6: FC0A::FF core: @@ -349,6 +351,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -372,6 +375,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -395,6 +399,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -418,6 +423,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -441,6 +447,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -464,6 +471,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -487,6 +495,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -510,6 +519,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -533,6 +543,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -556,6 +567,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -579,6 +591,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -602,6 +615,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -625,6 +639,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -648,6 +663,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -671,6 +687,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -694,6 +711,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -717,6 +735,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -740,6 +759,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -763,6 +783,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -786,6 +807,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -809,6 +831,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -832,6 +855,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -855,6 +879,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -878,6 +903,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -901,6 +927,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -924,6 +951,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -947,6 +975,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -970,6 +999,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -993,6 +1023,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1016,6 +1047,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1039,6 +1071,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1062,6 +1095,7 @@ configuration: - common - core bgp: + peer_in_bgp_confed: true asn: 65200 peers: 65100: @@ -1085,9 +1119,11 @@ configuration: - common - leaf bgp: - asn: 64600 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.128 - fc00::101 interfaces: @@ -1106,9 +1142,11 @@ configuration: - common - leaf bgp: - asn: 64610 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.130 - fc00::105 interfaces: @@ -1127,9 +1165,11 @@ configuration: - common - leaf bgp: - asn: 64620 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.132 - fc00::109 interfaces: @@ -1148,9 +1188,11 @@ configuration: - common - leaf bgp: - asn: 64630 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.134 - fc00::10d interfaces: @@ -1169,9 +1211,11 @@ configuration: - common - leaf bgp: - asn: 64640 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.136 - fc00::111 interfaces: @@ -1190,9 +1234,11 @@ configuration: - common - leaf bgp: - asn: 64650 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.138 - fc00::115 interfaces: @@ -1211,9 +1257,11 @@ configuration: - common - leaf bgp: - asn: 64660 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.140 - fc00::119 interfaces: @@ -1232,9 +1280,11 @@ configuration: - common - leaf bgp: - asn: 64670 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.142 - fc00::11d interfaces: @@ -1253,9 +1303,11 @@ configuration: - common - leaf bgp: - asn: 64680 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.144 - fc00::121 interfaces: @@ -1274,9 +1326,11 @@ configuration: - common - leaf bgp: - asn: 64690 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.146 - fc00::125 interfaces: @@ -1295,9 +1349,11 @@ configuration: - common - leaf bgp: - asn: 64700 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.148 - fc00::129 interfaces: @@ -1316,9 +1372,11 @@ configuration: - common - leaf bgp: - asn: 64710 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.150 - fc00::12d interfaces: @@ -1337,9 +1395,11 @@ configuration: - common - leaf bgp: - asn: 64720 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.152 - fc00::131 interfaces: @@ -1358,9 +1418,11 @@ configuration: - common - leaf bgp: - asn: 64730 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.154 - fc00::135 interfaces: @@ -1379,9 +1441,11 @@ configuration: - common - leaf bgp: - asn: 64740 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.156 - fc00::139 interfaces: @@ -1400,9 +1464,11 @@ configuration: - common - leaf bgp: - asn: 64750 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.158 - fc00::13d interfaces: @@ -1421,9 +1487,11 @@ configuration: - common - leaf bgp: - asn: 64760 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.160 - fc00::141 interfaces: @@ -1442,9 +1510,11 @@ configuration: - common - leaf bgp: - asn: 64770 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.162 - fc00::145 interfaces: @@ -1463,9 +1533,11 @@ configuration: - common - leaf bgp: - asn: 64780 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.164 - fc00::149 interfaces: @@ -1484,9 +1556,11 @@ configuration: - common - leaf bgp: - asn: 64790 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.166 - fc00::14d interfaces: @@ -1505,9 +1579,11 @@ configuration: - common - leaf bgp: - asn: 64800 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.168 - fc00::151 interfaces: @@ -1526,9 +1602,11 @@ configuration: - common - leaf bgp: - asn: 64810 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.170 - fc00::155 interfaces: @@ -1547,9 +1625,11 @@ configuration: - common - leaf bgp: - asn: 64820 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.172 - fc00::159 interfaces: @@ -1568,9 +1648,11 @@ configuration: - common - leaf bgp: - asn: 64830 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.174 - fc00::15d interfaces: @@ -1589,9 +1671,11 @@ configuration: - common - leaf bgp: - asn: 64840 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.176 - fc00::161 interfaces: @@ -1610,9 +1694,11 @@ configuration: - common - leaf bgp: - asn: 64850 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.178 - fc00::165 interfaces: @@ -1631,9 +1717,11 @@ configuration: - common - leaf bgp: - asn: 64860 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.180 - fc00::169 interfaces: @@ -1652,9 +1740,11 @@ configuration: - common - leaf bgp: - asn: 64870 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.182 - fc00::16d interfaces: @@ -1673,9 +1763,11 @@ configuration: - common - leaf bgp: - asn: 64880 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.184 - fc00::171 interfaces: @@ -1694,9 +1786,11 @@ configuration: - common - leaf bgp: - asn: 64890 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.186 - fc00::175 interfaces: @@ -1715,9 +1809,11 @@ configuration: - common - leaf bgp: - asn: 64900 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.188 - fc00::179 interfaces: @@ -1736,9 +1832,11 @@ configuration: - common - leaf bgp: - asn: 64910 + asn: 65300 + confed_asn: 65100 + confed_peers: 66000 peers: - 65100: + 66000: - 10.0.0.190 - fc00::17d interfaces: diff --git a/ansible/veos_vtb b/ansible/veos_vtb index 331e236e1cb..b4fea1c0288 100644 --- a/ansible/veos_vtb +++ b/ansible/veos_vtb @@ -38,6 +38,7 @@ all: - t0-backend - t0-88-o8c80 - dualtor + - dualtor-vpp - dualtor-56 - dualtor-120 - t2 @@ -53,6 +54,7 @@ all: - dualtor-mixed-56 - dualtor-mixed-120 - dualtor-aa + - dualtor-aa-vpp - dualtor-aa-56 - dualtor-aa-64 - dualtor-aa-120 @@ -133,7 +135,7 @@ all: mgmt_subnet_mask_length: 24 mgmt_subnet_v6_mask_length: 64 ansible_connection: multi_passwd_ssh - ansible_altpassword: YourPaSsWoRd + ansible_altpassword: "{{ lookup('env', 'SONIC_MGMT_SONIC_ALTPASSWORD') | default('YourPaSsWoRd', true) }}" children: vms_1: hosts: @@ -358,7 +360,7 @@ all: type: kvm hwsku: Force10-S6000 asic_type: vpp - serial_port: 9001 + serial_port: 9008 ansible_password: password ansible_user: admin vlab-vpp-02: @@ -367,7 +369,7 @@ all: type: kvm hwsku: Force10-S6000 asic_type: vpp - serial_port: 9001 + serial_port: 9091 ansible_password: password ansible_user: admin vlab-vpp-03: @@ -419,8 +421,8 @@ vm_host_1: hosts: STR-ACS-VSERV-01: ansible_host: 172.17.0.1 - ansible_user: use_own_value - vm_host_user: use_own_value + ansible_user: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_USER') | default('use_own_value', true) }}" + vm_host_user: "{{ lookup('env', 'SONIC_MGMT_VM_HOST_USER') | default('use_own_value', true) }}" vms_1: hosts: diff --git a/ansible/vtestbed.yaml b/ansible/vtestbed.yaml index a3cdf022d27..34e79529dbe 100644 --- a/ansible/vtestbed.yaml +++ b/ansible/vtestbed.yaml @@ -13,6 +13,7 @@ - vlab-01 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual switch vm - conf-name: vms-kvm-t0-csonic @@ -24,9 +25,14 @@ ptf_ipv6: fec0::ffff:afa:2/64 server: server_1 vm_base: VM0100 + vm_type: csonic dut: - vlab-01 inv_name: veos_vtb + # TODO(converged-peers): csonic neighbors are non-cEOS, so the converged + # (multi-VRF) peer model is gated off in conftest.py / testbed-cli.sh and + # never runs here. Re-enable once converged peers support csonic. + # use_converged_peers: True auto_recover: 'False' comment: Tests cSONiC virtual switch VMs without PortChannels @@ -58,6 +64,7 @@ - vlab-02 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual switch vm - conf-name: vms-kvm-t1-lag @@ -73,6 +80,7 @@ - vlab-03 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual switch vm - conf-name: vms-kvm-t0-2 @@ -104,6 +112,7 @@ - vlab-06 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Dual-TOR testbed - conf-name: vms-kvm-multi-asic-t1-lag @@ -134,6 +143,7 @@ - vlab-08 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests multi-asic virtual switch vm - conf-name: vms-kvm-t2 @@ -151,6 +161,7 @@ - vlab-t2-1-sup inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: T2 Virtual chassis with Multi-ASIC LCs - conf-name: vms-kvm-t0-3 @@ -350,6 +361,7 @@ - vlab-01 inv_name: veos_vtb auto_recover: False + use_converged_peers: True comment: Tests virtual switch vm as DPU - conf-name: vms-kvm-t1 @@ -524,6 +536,7 @@ - vlab-vpp-01 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual vpp switch vm - conf-name: vms-kvm-vpp-t1 @@ -539,6 +552,7 @@ - vlab-vpp-01 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual vpp switch vm - conf-name: vms-kvm-vpp-t0 @@ -554,11 +568,12 @@ - vlab-vpp-02 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: True comment: Tests virtual vpp switch vm - conf-name: vms-kvm-dual-vpp-t0-1 group-name: vms6-9 - topo: dualtor + topo: dualtor-vpp ptf_image_name: docker-ptf ptf: ptf-09 ptf_ip: 10.250.0.120/24 @@ -570,11 +585,12 @@ - vlab-vpp-04 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: False comment: Dual-TOR VPP testbed - conf-name: vms-kvm-dual-vpp-t0-2 group-name: vms6-10 - topo: dualtor-aa + topo: dualtor-aa-vpp ptf_image_name: docker-ptf ptf: ptf-10 ptf_ip: 10.250.0.121/24 @@ -587,4 +603,5 @@ - vlab-vpp-06 inv_name: veos_vtb auto_recover: 'False' + use_converged_peers: False comment: Active-Active Dual-TOR VPP testbed diff --git a/docs/testbed/README.testbed.cSONiC.md b/docs/testbed/README.testbed.cSONiC.md index d95b437cce6..c7ea35db2a3 100644 --- a/docs/testbed/README.testbed.cSONiC.md +++ b/docs/testbed/README.testbed.cSONiC.md @@ -25,6 +25,26 @@ cSONiC neighbors run the same SONiC software stack as the DUT, configured via CO The cSONiC testbed uses the `docker-sonic-vs` image as neighbor devices. +> **IMPORTANT — the stock/upstream `docker-sonic-vs` image is NOT sufficient.** +> A cSONiC neighbor must establish BGP and advertise LLDP exactly like a real +> SONiC device. This requires a `docker-sonic-vs` image that composes the +> control-plane features the neighbor depends on — at minimum **`bgpcfgd`** +> (from `docker-fpm-frr`, so BGP_NEIGHBOR entries in CONFIG_DB are translated +> into FRR config) and **`lldpd`/`lldpmgrd`** (from `docker-lldp`). The default +> upstream `docker-sonic-vs` (a swss+syncd test image) ships **without** +> `bgpcfgd` and `lldpd`, and its `start.sh` does not load `/var/sonic/config_db.json` +> or auto-start services under supervisord — so neighbors come up with no BGP +> sessions and no LLDP. Symptoms of using the wrong image: +> `sonic-db-cli CONFIG_DB PING` returns *"Connection refused / Cannot assign +> requested address"*, `start.sh` shows `FATAL Exited too quickly`, and +> `vtysh -c "show running-config"` has no `router bgp` stanza. +> +> Build the neighbor image from a `docker-sonic-vs` that includes the cSONiC +> feature composition (`_INCLUDE_DOCKER` for `docker-fpm-frr`, `docker-lldp`, +> `docker-teamd`, etc.) plus the `start.sh`/`supervisord` `dependent_startup` +> changes. See the `sonic-csonic-testbed` skill ("`_INCLUDE_DOCKER` — Feature +> Composition" and "Known Issues") for the full conversion checklist. + **Option 1: Download from Azure Pipelines** 1. Go to [SONiC Azure Pipelines](https://sonic-build.azurewebsites.net/ui/sonic/pipelines) 2. Find a recent successful build of `Azure.sonic-buildimage.official.vs` @@ -44,6 +64,142 @@ docker images | grep docker-sonic-vs The image should appear as `docker-sonic-vs:latest`. +### PortChannels / LACP on the host kernel + +cSONiC neighbors are **containers that share the host kernel**, so neighbor +PortChannels are realized by the *host* kernel — not by a private neighbor +kernel as with vSONiC (a KVM VM) or cEOS (Arista's own LAG implementation). +SONiC normally uses `teamd`/libteam, which needs the kernel **`team`** module. +The cSONiC image handles both cases automatically: + +- **`team` module present on the host** → the neighbor uses stock `teamd`, + exactly like production SONiC. Nothing extra to do. +- **`team` module absent** (e.g. some cloud kernels such as `*-azure` that ship + without it, or where Secure Boot blocks loading an out-of-tree `team`) → the + image **automatically falls back to the in-tree `bonding` driver in 802.3ad + (LACP) mode**. No host changes, no reboot, no module signing are required. + +#### Automatic bonding (802.3ad) fallback — recommended, zero host setup + +When `CSONIC_MODE` is active and the `team` driver cannot be loaded, the image's +`start.sh` builds each CONFIG_DB PortChannel with the `bonding` driver instead +of teamd: + +1. `teammgrd`/`teamsyncd` are stopped (they cannot work without `team`). +2. For each `PORTCHANNEL|*` it creates a bond: `ip link add type bond mode + 802.3ad miimon 100 lacp_rate fast`. +3. Each `PORTCHANNEL_MEMBER||` is **deleted from CONFIG_DB** so + `orchagent`/`saivs` release the member port's carrier (otherwise the port is + held `DOWN` while the broken teamd LAG stays pending), then the port is + enslaved to the bond. +4. `PORTCHANNEL` and `PORTCHANNEL_INTERFACE` are **kept** so `bgpcfgd` still sees + the local address and emits `router bgp`. The bond's IPs come from + `PORTCHANNEL_INTERFACE`. + +Linux bonding 802.3ad speaks standard LACP and bundles with the DUT's `teamd` +LACP. This is the default, validated path on hosts without `team`; **the DUT is +unchanged** and sees a normal `LACP(A)(Up)` PortChannel. No further action is +needed — just run `add-topo` + `deploy-mg`. + +> **Verifying the fallback:** on a neighbor, `ip -d link show PortChannel1` +> reports `bond mode 802.3ad`, `cat /proc/net/bonding/PortChannel1` shows MII +> status `up` with a non-zero partner MAC, and the syslog/`start.sh` log notes +> that the bonding fallback was used. On the DUT, `show interfaces portchannel` +> shows `LACP(A)(Up)` with the member `(S)` (Selected), and BGP reaches +> `Established`. + +#### Optional: load the kernel `team` module (use teamd instead of bonding) + +If you prefer the neighbor to run stock `teamd` (e.g. to exercise teamd-specific +behavior), load the `team` module on every VM host before deploying. This is +**not required** — the bonding fallback above covers PortChannel/LACP topologies +(T0/T1/T2) without it. + +```bash +lsmod | grep team # already loaded? +sudo modprobe team # load it +``` + +If `modprobe team` fails: +- **`Module team not found`** — the running kernel doesn't ship the module. + Install the matching extra-modules package (e.g. + `linux-modules-extra-$(uname -r)`), or use a kernel/distro that includes it. + Some cloud kernels (notably `*-azure`) omit `team` entirely. In that case + build it out-of-tree against the installed kernel headers (the team driver is + stable across point releases, so the mainline source for the matching + major.minor works): + + > **Important:** + > - Set `B=...linux/v.` to **your running kernel's + > major.minor** (`uname -r`), e.g. a `6.11.x` kernel needs `v6.11`. A + > mismatched source tree will fail to build or load. + > - This copies headers into the distro-managed `linux-headers-$(uname -r)` + > tree and drops modules under `/lib/modules/$(uname -r)`. These do **not** + > survive a kernel upgrade — after the host moves to a new kernel you must + > repeat the build (and the signing step below) for the new `uname -r`. + + ```bash + K=$(uname -r); mkdir -p /tmp/teambuild && cd /tmp/teambuild + B=https://raw.githubusercontent.com/torvalds/linux/v6.8 # MUST match your kernel's major.minor (uname -r) + for f in team team_mode_loadbalance team_mode_activebackup \ + team_mode_roundrobin team_mode_broadcast team_mode_random; do + curl -sSL -o $f.c "$B/drivers/net/team/$f.c" + done + # The team uapi/internal headers are not in the -headers package; fetch them: + curl -sSL -o if_team.h "$B/include/linux/if_team.h" + curl -sSL -o if_team_uapi.h "$B/include/uapi/linux/if_team.h" + sudo cp if_team.h /usr/src/linux-headers-$K/include/linux/if_team.h + sudo cp if_team_uapi.h /usr/src/linux-headers-$K/include/uapi/linux/if_team.h + printf 'obj-m += team.o team_mode_loadbalance.o team_mode_activebackup.o team_mode_roundrobin.o team_mode_broadcast.o team_mode_random.o\n' > Makefile + make -C /lib/modules/$K/build M=$PWD modules + sudo cp *.ko /lib/modules/$K/kernel/drivers/net/team/ # mkdir -p first if needed + sudo depmod -a $K + ``` + +- **`Key was rejected by service`** — **Secure Boot** is enabled and the module + is unsigned (you'll see this for any locally built/out-of-tree module; check + with `mokutil --sb-state` and `cat /sys/module/module/parameters/sig_enforce`). + You must sign the module with a key the firmware trusts, then **reboot once** + to enroll that key (there is no way to load an unsigned module under Secure + Boot without a reboot): + + ```bash + K=$(uname -r); cd /tmp/teambuild + # 1. Generate a Machine Owner Key (MOK) + openssl req -new -x509 -newkey rsa:2048 -keyout MOK.priv -outform DER \ + -out MOK.der -nodes -days 3650 -subj "/CN=csonic-team-module-signing/" + # 2. Sign every module + SF=/usr/src/linux-headers-$K/scripts/sign-file + for m in team team_mode_*; do sudo $SF sha256 MOK.priv MOK.der \ + /lib/modules/$K/kernel/drivers/net/team/${m%.ko}.ko; done + # 3. Stage the key for enrollment (choose a one-time password) + sudo mokutil --import MOK.der # prompts for a password + # 4. Reboot. At boot, shim's blue "MOK Manager" screen appears: + # select "Enroll MOK" -> "Continue" -> enter the password above -> reboot. + # 5. After reboot: + sudo modprobe team && lsmod | grep team + ``` + + Alternatively, disable Secure Boot entirely (Gen2/Azure VM setting or firmware + menu) — also a reboot. Once `team` loads, re-run `deploy-mg` so the neighbor + `teammgrd` can create the PortChannels. + +> **Note:** loading `team` is only needed if you specifically want `teamd` on the +> neighbor. If the module is absent the image automatically uses the bonding +> 802.3ad fallback (see above), so PortChannel topologies still work with no host +> changes. The symptoms below describe an **older image without the fallback**; +> the current image instead logs that it switched to bonding and the PortChannel +> bundles normally. + +> **Symptom of a missing `team` module:** `add-topo`/`deploy-mg` succeed, FRR +> comes up and `bgpcfgd` generates `router bgp`, but BGP stays `Active`/`Idle` +> and the DUT's `show interfaces portchannel` shows members `Dw`/`D` +> (deselected). The neighbor's `teammgrd` logs +> `Failed to create team device ... Operation not supported`, the +> `PortChannel1` netdev never appears, and the member port (`Ethernet1`) has no +> carrier. The BGP/LLDP control plane itself is fine — only LAG bundling is +> blocked. + ## Testbed Configuration ### vtestbed.yaml @@ -139,7 +295,14 @@ cSONiC neighbors support LACP PortChannels via CONFIG_DB: cSONiC neighbors are configured using CONFIG_DB + bgpcfgd, matching the production SONiC configuration path: -1. **Template**: `ansible/roles/sonic/templates/configdb-t0-leaf.j2` generates `config_db.json` +1. **Template**: `csonic_config.yml` selects the CONFIG_DB template with a + first-found fallback chain so any topology/role is supported: + `configdb-{topo}-{swrole}.j2` → `configdb-{swrole}.j2` → + `configdb-csonic.j2`. The generic `configdb-csonic.j2` is role-aware (it + maps `props.swrole` → DEVICE_METADATA `type`: `leaf`→`LeafRouter`, + `spine`→`SpineRouter`, `tor`→`ToRRouter`), so T1/T2/dualtor neighbor roles + render without a bespoke per-role file. Add a `configdb-{topo}-{swrole}.j2` + only when a role needs config that diverges from the generic template. 2. **Contents**: PORT, DEVICE_METADATA, LOOPBACK_INTERFACE, PORTCHANNEL, PORTCHANNEL_MEMBER, PORTCHANNEL_INTERFACE, INTERFACE, BGP_NEIGHBOR 3. **Deploy flow**: - `csonic_config.yml` renders the template and writes to `/var/sonic/config_db.json` (bind-mounted volume) diff --git a/docs/testplan/GH-packet-type-enhancement-test-plan.md b/docs/testplan/GH-packet-type-enhancement-test-plan.md new file mode 100644 index 00000000000..e8a350bd91c --- /dev/null +++ b/docs/testplan/GH-packet-type-enhancement-test-plan.md @@ -0,0 +1,229 @@ +# Generic Hash packet type enhancement Test Plan + +## Related documents + +| Document Name | Link | +| ------------ | ---- | +| SONiC Generic Hash | [hash-design.md](https://github.com/sonic-net/SONiC/blob/master/doc/hash/hash-design.md) | + + +## 1. Overview + +This enhancement extends generic hash features to support +* Native hash fields for RoCE traffic + - `SAI_NATIVE_HASH_FIELD_RDMA_BTH_OPCODE` + - `SAI_NATIVE_HASH_FIELD_RDMA_BTH_DEST_QP` + +* Per packet type ECMP/LAG hash configuration, leveraging SAI switch attributes including: + - `SAI_SWITCH_ATTR_ECMP_HASH_IPV4` + - `SAI_SWITCH_ATTR_ECMP_HASH_IPV4_IN_IPV4` + - `SAI_SWITCH_ATTR_ECMP_HASH_IPV6` + - `SAI_SWITCH_ATTR_LAG_HASH_IPV4` + - `SAI_SWITCH_ATTR_LAG_HASH_IPV4_IN_IPV4` + - `SAI_SWITCH_ATTR_LAG_HASH_IPV6` + - `SAI_SWITCH_ATTR_ECMP_HASH_IPV4_RDMA` + - `SAI_SWITCH_ATTR_ECMP_HASH_IPV6_RDMA` + - `SAI_SWITCH_ATTR_LAG_HASH_IPV4_RDMA` + - `SAI_SWITCH_ATTR_LAG_HASH_IPV6_RDMA` + +**Note:** This PR introduces the test plan document only. The actual test case implementation will be submitted in a follow-up PR. + +## 2. Requirements + +### 2.1 The enhanced feature supports: + +1. Per packet type ECMP and LAG hash configuration (IPv4, IPv6, IPv4-in-IPv4, RDMA, etc.). +2. The packet type hash takes precedence when configured; otherwise, the global switch hash is applied. +3. Reboot/reload with state persistence for all packet type configs. + +### 2.2 Supported commands: + +1. `config` commands to set per packet type hash for ECMP/LAG. +2. `show` commands display per packet type and global hash configuration/capability. + +## 3. Scope + +1. Verify per packet type hash config can be independently managed and affects ECMP/LAG distribution only for matching traffic. +1. Detailed error-handling and capability-driven behavior (e.g., unsupported packet types or hash fields) are validated by separate mock/unit tests and are intentionally not covered by this PTF test plan. +1. Algorithm behavior is covered by the base generic-hash tests and none in this test plan. + +### 3.1 Scale / Performance + +No additional scale requirements compared to generic hash. + +### 3.2 CLI Commands + +#### 3.2.1 Config +The following command can be used to configure generic hash with packet-type support: +``` +config +|--- switch-hash + |--- global + |--- ecmp-hash [--packet-type --action ] ARGS + |--- lag-hash [--packet-type --action ] ARGS + |--- ecmp-hash-algorithm ARG + |--- lag-hash-algorithm ARG +``` + + +#### 3.2.2 Show +The following command shows switch hash global configuration: +``` +show +|--- switch-hash + |--- global [packet-type ] + |--- capabilities +``` + +#### 3.2.3 Supported packet types +| Packet Type | Description | +|------------------|-----------------| +| ipv4 | IPv4 packets | +| ipv6 | IPv6 packets | +| ipnip | IPv4-in-IPv4 encapsulated packets (CLI packet type for SAI `IPV4_IN_IPV4`) | +| ipv4-rdma | RDMA over IPv4 packets | +| ipv6-rdma | RDMA over IPv6 packets | +| all | Show all packet type configurations (for show command only) | + + +**Note:-** +- _pkt-type (Supported values, CLI names):_ + - _all, ipv4, ipv6, ipnip, ipv4-rdma, ipv6-rdma_ + - _CLI packet type `ipnip` maps to the SAI packet type `IPV4_IN_IPV4`_ +- _In config command:_ + - _`--packet-type `: Optional parameter; if omitted, updates global hash_ + - _`--action `: Required when `--packet-type` is specified_ + - _`add`: Adds the specified hash fields for the given packet-type hash. Duplicate fields are ignored_ + - _`del`: Removes the specified hash fields for the given packet-type hash. If no hash fields are provided, the given packet-type hash is deleted_ + +- _In show command:_ + - _`packet-type ` is optional; `all` packet-type is valid only for show_ + - _If pkt-type omitted: Shows global hash configuration/capabilities_ + - _If pkt-type is all: Shows all packet type hash configuration/capabilities_ + + +### 3.3 CLI usage examples +1. config switch-hash global ecmp-hash 'SRC_MAC' 'ETHERTYPE' +1. config switch-hash global ecmp-hash --packet-type ipv4 --action add 'SRC_IP' 'DST_IP' +1. config switch-hash global lag-hash --packet-type ipv6-rdma --action add 'RDMA_BTH_OPCODE' 'RDMA_BTH_DEST_QP' +1. config switch-hash global ecmp-hash --packet-type ipv4 --action del +1. show switch-hash global packet-type ipv4 +1. show switch-hash global packet-type all + +### 3.4 Supported topology +1. The test plan targets both t0 and t1 topologies. +1. Tests requiring multi-member LAGs/portchannels and multiple ECMP next-hops will be skipped if the topology does not provide the necessary resources. + +## 4 Test Cases for per packet type hash enhancement + +### 4.1 Test case list + +| No. | Test Name | Purpose | +|-----|----------------------------------------------|-----------------------------------------------------------------------| +| 1 | test_rdma_hash_field_distribution | Verify RDMA hash field impact on traffic distribution | +| 2 | test_hash_field_distribution_ip | Verify non-RDMA IP hash field impact on traffic distribution | +| 3 | test_pkt_type_hash_priority_and_override | Priority/override between default and per-pkt-type hash | +| 4 | test_pkt_type_hash_config_persistence_reload | Persistence of pkt_type_hash config after reboot/reload for ECMP/LAG | +| 5 | test_pkt_type_hash_warm_boot | Validate warm boot with packet type hash for ECMP/LAG | +| 6 | test_pkt_type_hash_fast_boot | Validate fast boot with packet type hash for ECMP/LAG | + +### 4.2 Test case descriptions + +**Note:-** _Tests will be repeated for different packet types (where supported by the platform)_ + +#### 1. test_rdma_hash_field_distribution +--- +**Purpose:** Configure RDMA fields (`RDMA_BTH_OPCODE`, `RDMA_BTH_DEST_QP`) for RDMA packet types; send test traffic and verify egress distribution changes per field. + +**Note:** Before implementing this test, ensure that the RDMA hash field constants (`RDMA_BTH_OPCODE`, `RDMA_BTH_DEST_QP`) and validation logic are added to the test framework. + +**Steps:** +1. Configure RDMA fields: + - `config switch-hash global ecmp-hash --packet-type ipv6-rdma --action add 'DST_MAC' 'RDMA_BTH_OPCODE' 'RDMA_BTH_DEST_QP'` + - `config switch-hash global lag-hash --packet-type ipv6-rdma --action add 'DST_MAC' 'RDMA_BTH_OPCODE' 'RDMA_BTH_DEST_QP'` +1. Generate RDMA-over-IPv6 packets varying BTH Opcode and Dest_QP. +1. Observe load-balancing across ECMP/LAG paths. + +**Expected Result:** Egress ports vary based on RDMA fields and distribution observed across members. + +#### 2. test_hash_field_distribution_ip +--- +**Purpose:** Configure standard (non-RDMA) IP hash fields for a supported packet type (IPv4 or IPv6) and verify that traffic distribution across ECMP/LAG paths varies based on the configured hash fields. The test is repeated for ECMP-only, LAG-only, and ECMP+LAG modes; in ECMP+LAG mode both are configured with the same field set (unified hash model). + +**Steps:** +1. Query platform capabilities from STATE_DB and select the first supported non-RDMA IP packet type (IPv4 preferred, then IPv6). Skip if none are supported. +1. For LAG-only or ECMP+LAG modes, verify that a multi-member LAG/portchannel exists; skip otherwise. +1. Determine the hash field list by intersecting a base set of IP fields (`SRC_IP`, `DST_IP`, `L4_SRC_PORT`, `L4_DST_PORT`, `DST_MAC`) with the ASIC-supported hash fields for the selected mode. Skip if no compatible fields are available. +1. Configure per-packet-type hash on the DUT for the selected mode: + - ECMP-only: `config switch-hash global ecmp-hash --packet-type --action add ` + - LAG-only: `config switch-hash global lag-hash --packet-type --action add ` + - ECMP+LAG: configure both ECMP and LAG with the same fields. +1. Verify the packet-type hash configuration is correctly reflected in Config DB. +1. Generate IP test traffic (matching the selected packet type) varying a chosen hash field (preferring `SRC_IP`) and send it through the PTF test framework. +1. Observe egress port distribution across ECMP next-hops and/or LAG members. + +**Expected Result:** Egress ports vary based on the configured IP hash fields, demonstrating that per-packet-type hash configuration correctly influences traffic distribution for non-RDMA IP traffic across ECMP and/or LAG paths. + +#### 3. test_pkt_type_hash_priority_and_override +--- +**Purpose:** Configure default hash and per packet-type hash; generate matching/non-matching traffic and verify per packet-type config is prioritized for that traffic, default used otherwise. + +**Steps:** +1. Configure default ECMP hash: `config switch-hash global ecmp-hash 'SRC_MAC' 'ETHERTYPE'` +1. Select two supported packet types (e.g., packet-type 1: IPv4, packet-type 2: IPv6) and configure unique hashes for each: +1. The hash fields for packet-type can be selected randomly. +1. For packet-type 1 (e.g., IPv4): + - config switch-hash global ecmp-hash --packet-type ipv4 --action add + - Example: config switch-hash global ecmp-hash --packet-type ipv4 --action add 'SRC_IP' 'DST_IP' +1. Generate traffic corresponding to the selected packet-types. + - Example: Send both IPv4 and IPv6 packets +1. Observe hash result. +1. Repeat the test case for LAG + +**Expected Result:** Packet-type 1 traffic follows per packet-type hash and packet-type 2 traffic continues to use the default global hash. + +#### 4. test_pkt_type_hash_config_persistence_reload +--- +**Purpose:** Configure various packet-type hashes, reload/reboot the switch and ensure configuration and data plane behavior persist. + +**Steps:** +1. Configure ECMP and LAG hashes for multiple pkt-types. +1. Send relevant traffic continuously. +1. Save config and reboot. +1. Post reboot, run: `show switch-hash global packet-type all` +1. Verify data-plane behavior remains consistent after reboot. + +**Expected Result:** All per packet-type configs are preserved after reboot. + +#### 5. test_pkt_type_hash_warm_boot +--- +**Purpose:** Ensure that both ECMP and LAG packet-type hash configurations for selected packet types persist across a warm boot. +**Steps:** +1. Configure ECMP/LAG with global and packet-type-specific hashes for at least two supported packet types (for example: `ecmp_hash_ipv4`, `ecmp_hash_ipv6`, `ecmp_hash_ipv4_rdma`, `ecmp_hash_ipv6_rdma`). +1. Send relevant traffic continuously for each selected packet type. +1. Trigger a warm boot, and after boot, verify: + - Packet-type hash behavior for ECMP/LAG is preserved for all selected packet types. + - There should be no traffic loss. + +**Expected Result:** Packet-type hash behavior for ECMP/LAG is preserved for all selected packet types with no traffic loss. + +#### 6. test_pkt_type_hash_fast_boot +--- +**Purpose:** Ensure that both ECMP and LAG packet-type hash configurations for all supported packet types persist across fast boot. + +**Steps:** +1. Configure ECMP/LAG with global hash settings and per-packet-type hash configurations for all supported packet types (for example, IPv4, IPv4-in-IPv4, IPv6, and RDMA-related packet types, as applicable on the DUT). +1. Send continuous traffic for each configured packet type across the ECMP/LAG members. +1. Trigger a fast boot, then after the device is back online, verify: + - For each configured packet type, packet-type hash behavior for ECMP/LAG is preserved. + - Traffic drop for each flow should not exceed 30 seconds. + +**Expected Result:** For all configured/supported packet types, packet-type hash behavior for ECMP/LAG is preserved across fast boot, with any traffic loss per flow limited to less than 30 seconds. + + +### General Verification Points +- Packet-type hash configuration should persist across both warm and fast boot. +- State DB and config DB values must restore correctly. +- Hash functionality should match pre-boot state. +- No unexpected errors in logs. + diff --git a/docs/testplan/live_addon_docker_hld.md b/docs/testplan/live_addon_docker_hld.md new file mode 100644 index 00000000000..8428165a93d --- /dev/null +++ b/docs/testplan/live_addon_docker_hld.md @@ -0,0 +1,293 @@ +# Live-Addon Docker Test Framework Test Plan and High-Level Design + +## 1. Problem statement + +SONiC platforms may ship a vendor-specific **live-addon** container (diagnostics, health agent, +utility services) alongside the base image. The container is installed with `docker run`, not +`sonic-package-manager`. + +The sonic-mgmt test module `tests/live_addon_docker/` automates: + +- Resolving the live-addon image (registry pull, tarball, or pre-loaded image on the DUT) +- Starting the container from vendor JSON configuration +- Post-start validation after every `docker run` (single instance, logs or supervisord poll) +- HTTP health and survival across `config reload` +- Teardown checks (no new cores, container removed, syslog spot-check) + +The module name is vendor-neutral. Vendor-specific names (image repo, container name, mount paths) +live in per-ASIC JSON files under `tests/live_addon_docker/files/`. + +## 2. Scope + +| In scope | Out of scope | +|----------|--------------| +| `docker pull` / `docker load` / `docker run` on the DUT | `sonic-package-manager` install paths | +| HTTP health endpoint probe from the DUT | Building or publishing docker images | +| Post-start checks via `verify_live_addon_post_start` (logs and/or supervisord) | Shared tarball distribution between vendors | +| Optional `version_matrix` skip for image vs SONiC compatibility | | +| Registry override per test run (`--live_addon_docker_registry`) | | + +**Platform filter:** `asic_type=cisco-8000` only (see +`tests/common/plugins/conditional_mark/tests_mark_conditions_live_addon_docker.yaml`). VS/KVM is +skipped. Additional asic types can be added when supported. + +## 3. Test coverage + +Post-start validation is **not** duplicated in pytest cases; it runs in the module fixture and inside +`run_config_reload_live_addon_start_reload_health` on each `docker run`. + +| Test | Validates | +|------|-----------| +| `test_live_addon_docker_health_http` | HTTP `/health` returns expected status within probe timeout | +| `test_live_addon_docker_health_after_config_reload_cycle` | Stop container → `config reload` → `docker run` + full post-start → `config reload` → teardown + `docker run` + restart post-start (120s supervisord) → HTTP health | + +**Module fixture** `live_addon_docker_setup_teardown`: install once per module, `docker run`, +`verify_live_addon_post_start` (full readiness), yield `(duthost, cfg)`, then teardown and +post-teardown checks. + +**Typical runtime (Cisco):** first start may wait up to **900s** for startup logs; config-reload +cycle adds another full post-start plus a **120s** supervisord poll on restart; HTTP health polls +up to **900s** when needed. + +**Topology:** tests are marked `pytest.mark.topology("any")`. Use `-t any` or `-t t1,any` with +`run_tests.sh` (a bare `-t t1` skips these tests). + +## 4. Repository layout + +``` +tests/live_addon_docker/ +├── conftest.py # pytest options and fixtures +├── live_addon_docker_helpers.py # DUT command assembly and validation +├── test_live_addon_docker.py # health-focused test cases +└── files/ + └── cisco-8000_live_addon_docker.json # Cisco 8000 default config +``` + +Default config path when `--live-addon-docker-config` is not set: + +``` +tests/live_addon_docker/files/_live_addon_docker.json +``` + +Example: `asic_type=cisco-8000` → `cisco-8000_live_addon_docker.json`. + +## 5. Image and container naming + +### 5.1 Docker image repository (SONiC ACR convention) + +Repository name follows the same pattern as `docker-syncd-cisco` and `docker-gbsyncd-cisco`: + +``` +docker-live-addon-[:tag] +``` + +| Field | Cisco 8000 example | +|-------|-------------------| +| `vendor` (JSON) | `cisco` | +| ACR repository | `docker-live-addon-cisco` | +| `docker_run.image_ref` | `docker-live-addon-cisco:latest` | +| Tarball filename | `docker-live-addon-cisco.gz` | + +The `vendor` field drives `docker_run.image_ref` when only `image_tag` is set. An explicit +`image_ref` in JSON must use the `docker-live-addon-` repository for registry pull to +succeed. + +### 5.2 Container name (vendor-specific) + +The running container name is **not** normalized across vendors. Cisco keeps the manifest label +name: + +```json +"docker_run": { "container_name": "cisco-utility" } +"validation": { "docker_container_name": "cisco-utility" } +``` + +Other vendors set their own `container_name` in JSON (for example `acme-diagnostics`). + +## 6. Image resolution flow + +Registry pull is attempted first when a registry host is available. On failure, the framework +falls back to tarballs or an image already on the DUT. + +```mermaid +flowchart TD + A[Test start] --> B{Registry host configured?} + B -->|yes| C[docker pull on DUT] + C -->|success| D[docker tag to image_ref] + C -->|fail| E{Tarball on DUT admin home?} + B -->|no| E + E -->|yes| F[docker load from tarball] + E -->|no| G{Tarball on test runner?} + G -->|yes| H[copy to DUT tmp and docker load] + G -->|no| I{Image already on DUT?} + I -->|yes| J[use existing image] + I -->|no| K[skip test] + D --> L[version_matrix check] + F --> L + H --> L + J --> L + L -->|compatible| M[docker run] + L -->|incompatible| N[skip test] + M --> O[verify_live_addon_post_start] + O --> P[Run tests] + P --> Q[Teardown and post checks] +``` + +Registry host comes from Ansible `docker_registry_host` or pytest `live_addon_docker_registry` +(see §7). Tarball path on the DUT is `dut_tarball_home` plus `tarball_filename` from JSON. + +**Pull tag selection:** + +1. `--live_addon_docker_image_tag` if set (CI build id) +2. Else tag from `docker_run.image_ref` when not `latest` +3. Else `duthost.os_version` (same convention as syncd-rpc / `swap_syncd`) + +## 7. Pytest CLI parameters + +| Option | Purpose | +|--------|---------| +| `--live-addon-docker-config` | Override path to vendor JSON | +| `--live-addon-docker-tarball` | Path to `.gz` on the test runner | +| `--live_addon_docker_registry` | Registry host for pull (overrides Ansible `docker_registry_host` for this module) | +| `--live_addon_docker_image_tag` | Image tag for pull and `docker_run.image_ref` | +| `--public_docker_registry` | Use `public_docker_registry_host` without login (same as `swap_syncd`) | + +**Example via `run_tests.sh`:** + +```bash +cd tests +./run_tests.sh \ + -n \ + -d \ + -t any \ + -c live_addon_docker/test_live_addon_docker.py \ + -i ../ansible/veos \ + -e "--live_addon_docker_registry=myacr.azurecr.io --live_addon_docker_image_tag=kube-20260527-202505-amd64" +``` + +Each vendor or MSFT can point at their own container registry without sharing tarballs. + +## 8. Vendor JSON schema + +Required top-level fields: + +| Key | Description | +|-----|-------------| +| `vendor` | Short vendor id; drives `docker-live-addon-` repo name | +| `docker_run` | `image_ref`, `container_name`, `cli_args` for `docker run` | +| `health` | HTTP probe: `port`, `url_path`, `bind_host`, `expect_http_code`, optional wait/probe timeouts | +| `validation` | `docker_container_name`; optional `expected_processes`, `startup_log`, `max_running_instances` | +| `tarball_filename` | `.gz` name for tarball fallback paths | + +Optional: + +| Key | Description | +|-----|-------------| +| `version_matrix` | Skip when live-addon `package.version` and DUT SONiC are not listed as compatible | +| `candidate_image_refs` | Extra refs for `docker image inspect` on the DUT | +| `dut_tarball_home` | DUT path for pre-staged tarball (default `/home/admin`) | + +Commands executed on the DUT are built only from this JSON (`live_addon_docker_helpers.py`). +Post-teardown checks (cores, syslog grep, container absent) are fixed in code. + +### Post-start validation (`verify_live_addon_post_start`) + +Called after **every** `docker run` (module fixture, config-reload steps, restarts). Always runs: + +1. Single-instance check (`max_running_instances`, exact container name, optional image ancestor filter) + +Then readiness (both may run when configured): + +| Mode | Startup logs | Processes | +|------|--------------|-----------| +| `full_readiness=True` + `startup_log` configured | Poll until patterns match | One-shot ``supervisorctl`` assert after logs pass | +| `full_readiness=True`, no `startup_log` | Skipped | Poll up to 120s | +| `full_readiness=False` (restart) | Skipped (avoids long log wait) | Poll up to 120s | + +Log patterns and ``supervisorctl`` are complementary: logs prove supervisord/diagnostic startup +lines; ``supervisorctl`` confirms program state directly. + +**`validation` post-start fields (Cisco example — matches `cisco-8000_live_addon_docker.json`):** + +```json +"validation": { + "docker_container_name": "cisco-utility", + "expected_processes": ["start", "health-monitor", "health-server"], + "startup_log": { + "wait_seconds": 900, + "poll_interval_seconds": 30, + "session_start_pattern": "supervisord started with pid", + "required_patterns": [ + "supervisord started with pid", + "success: start entered RUNNING state", + "success: health-monitor entered RUNNING state", + "success: health-server entered RUNNING state", + "diagnostic is running now" + ], + "forbidden_patterns": ["spawnerr", "exited too quickly", "entered FATAL state", "BACKOFF", "[FAILED]"] + } +} +``` + +- `expected_processes` — supervisord program names for restart polling and vendors without + `startup_log`. Names normalize `_` vs `-`. Omit to use code defaults; set `[]` to skip. +- `max_running_instances` — default **1**. Assert exact running count by name, at most that many + in `docker ps -a`, and (unless `enforce_single_image_instance` is false) running count from + `docker_run.image_ref`. +- `startup_log` — **vendor-specific**. Poll `docker logs` every `poll_interval_seconds` until all + `required_patterns` appear. Fail on `forbidden_patterns`. Omit or use empty `required_patterns` + to skip. + - **Timing:** code default `wait_seconds` is **120s**. Cisco sets **900s** because online + diagnostic waits for syncd uptime before `diagnostic is running now` can appear. + - **Process poll (restarts):** code default **120s** / **30s** interval (`DEFAULT_PROCESS_*`); + independent of `startup_log.wait_seconds` and `health.probe_timeout_seconds`. + - **Poll interval:** default **30s** when omitted. Intermediate poll status is DEBUG only; log + text prints once on success or final timeout (last 6000 chars on failure). + - **Log scope:** ``docker logs --since `` from ``docker inspect``. Optional + ``session_start_pattern`` slices from the last matching line if ``StartedAt`` is unavailable. + +## 9. Version matrix + +Omit `version_matrix`, set it to `null`, or use `[]` to disable the check. + +The check runs after the image is on the DUT (`docker load` or already present) and before +`docker run`. It uses `docker image inspect` and reads `package.version` from label +`com.azure.sonic.manifest` (not the Docker `:tag`). + +Each row may include: + +- `utility_package_version_glob` / `utility_image_version_glob` — fnmatch on `package.version` +- `compatible_sonic_globs` — fnmatch on `duthost.os_version`, `sonic_release`, or `show version` first line + +**Skip conditions:** + +- No row matches `package.version` (including when the manifest label is missing) +- Matching row has no `compatible_sonic_globs` +- DUT SONiC does not match any allowed glob + +Images built without `com.azure.sonic.manifest` skip until the build pipeline adds standard SONiC +docker labels. + +Example: + +```json +"version_matrix": [ + { + "utility_package_version_glob": "202405*", + "compatible_sonic_globs": ["202411*", "202505*"] + } +] +``` + +## 10. Adding a new vendor / ASIC + +1. Add `tests/live_addon_docker/files/_live_addon_docker.json`. +2. Set `vendor`, `docker_run.container_name`, `health` port/path, and vendor-specific `cli_args` mounts. +3. Ensure ACR publishes `docker-live-addon-:` with `com.azure.sonic.manifest` when using `version_matrix`. +4. Extend `tests_mark_conditions_live_addon_docker.yaml` if the ASIC should not be skipped. +5. Run with `--live_addon_docker_registry` pointing at the vendor CR. + +## 11. Assumptions and constraints + +- DUT has Docker and network access to the chosen registry (or a pre-staged image/tarball). +- Registry credentials come from Ansible `docker_registry_*` in testbed creds unless overridden by CLI. diff --git a/docs/testplan/snappi/assets/snake_topo.png b/docs/testplan/snappi/assets/snake_topo.png new file mode 100644 index 00000000000..d53bc15f23e Binary files /dev/null and b/docs/testplan/snappi/assets/snake_topo.png differ diff --git a/docs/testplan/snappi/srv6_performance_test.md b/docs/testplan/snappi/srv6_performance_test.md new file mode 100644 index 00000000000..1a3a6f466b4 --- /dev/null +++ b/docs/testplan/snappi/srv6_performance_test.md @@ -0,0 +1,108 @@ +# SONiC Switch SRv6 Dataplane Performance Test + +- [Test Objective](#test-objective) +- [Test Setup](#test-setup) +- [Test Parameters](#test-parameters) +- [Test Steps](#test-steps) +- [Metrics to Collect](#metrics-to-collect) + +## Test Objective + +This test aims to assess the data-plane performance of the SRv6 forwarding function of a SONiC switch. + +## Test Setup + +### Network Topology Setup + +The test is designed to be topology-agnostic. +The recommended topology to use includes: +1. nut-2tiers as defined in [nut-2tiers.yml](../../../ansible/vars/nut_topos/nut-2tiers.yml). The example topology figure is shown in [NUT doc](../../testbed/README.testbed.NUT.md). +2. Snake topo as shown in ![Snake Topology](./assets/snake_topo.png) + +Note: If we are using breakout ports, when we connect the links between traffic generators and switches in the topology, we should pay attention to the port distribution of the switch ASIC. Ideally, we should run the test on two instances of every type of the topology. These two instances should respectively follow the two port distributions below: +1. Gather all the ports under test in the same breakout groups of front-panel physical ports. For example, when using 8-to-1 breakout with 8 ports under test, we should connect ports etp1a~etp1h +2. Spread the ports under test to different breakout port groups that span across multiple front-panel physical ports. For example, when using 8-to-1 breakout with 8 ports under test, we should connect ports etp[1-8]a. + +### Network Configuration + +Generally, the DUT should have SRv6 and route configurations as follows: +- Every Device Under Test(DUT) should be configured with a number of SRv6 SIDs up to the maximum number of parallel links between DUT and the neighbors. If using fcbb:bbbb:: as the locator block, the SRv6 SIDs of a switch with I as device index in the topo and Q as the number of maximum parallel links can be configured to be fcbb:bbbb:hex(I << 8 + 1)::/48 ~ fcbb:bbbb:hex(I << 8 + Q)::/48. +- Every Traffic Generator(TG) should also be configured a number of SRv6 SIDs each of which corresponds to a link between the traffic generator and the DUT. If using fcbb:bbbb:: as the locator block, the SRv6 SIDs of a traffic generator, with I as device index and N ports connected to the switch, can be as fcbb:bbbb:hex(I << 8 + 1)::/48 ~ fcbb:bbbb:hex(I << 8 + N)::/48. +- The DUT should have a static route entry configured for each SRv6 SID that its neighbors (including both DUTs and TGs) have. + +For the snake topology, the network configuration will be more complex because: +- Two router interfaces in SONiC cannot belong to the same subnet, so the interconnected pairs of switch ports need to be configured with different VRFs. +- By default all SONiC router interfaces use the same router mac addresses but two interconnected interfaces cannot have the same mac address. One way to resolve this issue is to put every router interface in a VLAN and then customize the MAC address of the VLAN interface. +- Different hardware platforms have different compatabilities between SRv6 and VRF. If the switch ASIC being used does not support SRv6 MY_SID lookup in non-default VRFs, the user should put the receiving interface in the default VRF and then set the successive static route to use an interface in a non-default VRF. To allow a single interface to carry both ingress and egress traffic, the user may configure two subinterfaces on it and then put one of the subinterface in the default VRF for ingress traffic while keeping the other subinterface in a non-default VRF for egress traffic. + + +### Traffic Generation Configuration + +The traffic generators should be configured to send traffic with SRv6 SIDs in IPv6 header and optionally Segment Routing Header. +The exact way of configuring the SRv6 SIDs in the header depends on the topology and the traffic path the users want to test. + +However, there are three principles to follow: +1. We should split the ports of the traffic generators into two groups (e.g. first half vs last half) with equal number of ports. +2. Each port of the traffic generators should mutually exclusively communicate with a single port in the other group in a bidrectional way. +3. Every pair of ports should communicate using a SRv6 path (by specifying SRv6 SID list in IPv6 header) that does not share any link with any other pair of ports so that the network is congestion free by design. + +We give two examples for nut2tiers topology and snake topology here: + +#### SRv6 SIDs configuration for `nut-2tiers` Topology + +For a NUT which have M T0 devices with M traffic generators that has N ports, the SRv6 paths used by each traffic generator port can be calculated as follows: +- The ports of the i-th (0 <= i < M/2) traffic generator (in the first group) can use SRv6 paths: + - fcbb:bbbb:hex(i)01:hex(16M)01:hex(M/2 + i)01:hex(M/2 + i)hex(N + 1):: + - fcbb:bbbb:hex(i)02:hex(16M)02:hex(M/2 + i)02:hex(M/2 + i)hex(N + 2):: + - ... + - fcbb:bbbb:hex(i)10:hex(16M)hex(N):hex(M/2 + i)10:hex(M/2 + i)hex(2N):: +- The ports of the i-th (M/2 <= i < M) traffic generator (in the second group) can use SRv6 paths: + - fcbb:bbbb:hex(i)01:hex(16M)01:hex(i - M/2)01:hex(i - M/2)hex(N + 1):: + - fcbb:bbbb:hex(i)02:hex(16M)02:hex(i - M/2)02:hex(i - M/2)hex(N + 2):: + - ... + - fcbb:bbbb:hex(i)10:hex(16M)hex(N):hex(i - M/2)10:hex(i - M/2)hex(2N):: + +#### SRv6 SIDs configuration for Snake Topology +Supposedly, there are two groups of traffic generator ports on both sides of the 'Snake'. +If one group of traffic generator ports consists of N ports, for a port of indexed by i (1 <= i <= N), the packet sent by the traffic generator should have SRv6 SID list in IPv6 header (and potentially Segment Routing Header) as follows: +- fcbb:bbbb:i00:i:hex(N + i):hex(2N + i)...hex(MN + i):hex(N+i)00::, note: hex(N+i)00 refers to the SRv6 SID of the receiving traffic generator. + +To maximize the stress on the DUT, the i-th port of the traffic generators on the other side of the topology should send packets to the DUT with SRv6 SID list as follows: +- fcbb:bbbb:hex(N+i)00:hex(MN + i):hex((M-1)N + i):...:i:i00::, note: this is essentially the reverse of the SID list used by the other side. + +### Metrics Monitoring + +The test should perform the following metrics monitoring: +- Collects all metrics listed in [Switch Capability Test](./switch_capacity_test.md), [Switch Packet Drop Threshold Test](./switch-packet-drop-threshold-tests.md) and [Switch latency Tests](./switch-latency-tests.md) periodically from switches during the test. +- Collects additional metrics listed in [Metrics to Collect](#metrics-to-collect) periodically from switches during the test. + +## Test Parameters + +- `test_duration`: The duration of the test in minutes, which supports 1min, 5min, 15mins, 60mins, 1day and 2days. +- `packet_size`: The size of the packets in bytes to be sent in the traffic, which supports 128, 256, 4096, and mix of packet size (In the mix, the 128 packet size should always only occupy 1% of the traffic). +- `collect_interval`: The interval between two metrics collection operations on the switch. + +## Test Steps + +1. For each combination of test parameters, start the traffic generator to generate traffic according to the parameters provided. +2. Start the monitoring thread to collect metrics from all SONiC devices in the testbed. +3. Wait until the test to be completed. +4. Stop the traffic generator. + +## Additional Metrics to collect + +During this test, we are going to collect the following additional metrics from the SONiC device in the testbed: + +### SRv6 MY_SID Metrics + +The `show srv6 stat` command is used on the switch to retrieve the packets and bytes counter for every SRv6 MY_SID entry configured on the device. The following labels are expected to be provided: + +| Metrics Label | Label Key in DB | Example Value | Description | +|-----------------------------------|------------------|-----------------|-------------------| +| `METRIC_LABEL_DEVICE_ID` | device.id | switch-A | Switch Identifier | +| `METRIC_LABEL_DEVICE_SRV6_MY_SID` | device.srv6.my_sid | fcbb:bbbb:1::/48 | IP Prefix of the SRv6 SID entry | + +| User Interface Metric Name | Metric Name in DB | Example Value | +|----------------------------------------|---------------------------------|---------------------| +| `METRIC_NAME_SRV6_MY_SID_BYTES` | srv6.my_sid.rx.bytes | 10000 | +| `METRIC_NAME_SRV6_MY_SID_PACKETS` | srv6.my_sid.rx.packets | 2 | \ No newline at end of file diff --git a/docs/testplan/transceiver/diagrams/file_organization.md b/docs/testplan/transceiver/diagrams/file_organization.md index d0393868d05..9843b143d45 100644 --- a/docs/testplan/transceiver/diagrams/file_organization.md +++ b/docs/testplan/transceiver/diagrams/file_organization.md @@ -34,7 +34,7 @@ ansible/files/transceiver/inventory/ │ ├── system/ # Same shape as eeprom/ │ ├── physical_oir/ │ ├── remote_reseat/ -│ ├── cdb_fw_upgrade/ +│ ├── cdb_firmware_upgrade/ │ ├── dom/ │ ├── vdm/ │ ├── pm/ @@ -204,13 +204,13 @@ tests/transceiver/ │ ├── test_link_stability.py # TC 6: Link stability monitoring │ └── test_power_cycle_stress.py # TC 7: Power cycle stress test │ -├── cdb_fw_upgrade/ +├── cdb_firmware_upgrade/ │ ├── __init__.py -│ ├── conftest.py # CDB FW upgrade-specific fixtures; autouse fixture requests +│ ├── conftest.py # CDB firmware upgrade-specific fixtures; autouse fixture requests │ │ # presence_verified, links_verified from top-level conftest.py -│ │ # (gold FW is CDB FW's own reportable test, so that gate is +│ │ # (gold FW is CDB firmware upgrade's own reportable test, so that gate is │ │ # intentionally not consumed.) -│ └── test_fw_upgrade.py # CDB FW upgrade test cases; includes gold FW check +│ └── test_fw_upgrade.py # CDB firmware upgrade test cases; includes gold FW check │ # (reportable test case; calls common/prerequisites.py::check_gold_firmware) │ ├── port_config/ diff --git a/docs/testplan/transceiver/test_plan.md b/docs/testplan/transceiver/test_plan.md index 63aa1de4d04..7f88bb9b856 100644 --- a/docs/testplan/transceiver/test_plan.md +++ b/docs/testplan/transceiver/test_plan.md @@ -121,13 +121,13 @@ Each gate is a session-scoped pytest fixture defined in [`tests/transceiver/conf | System (`system.json`) | ✅ | ✅ | ✅ | | Physical OIR (`physical_oir.json`) | ✅ | ✅ | ✅ | | Remote Reseat (`remote_reseat.json`) | ✅ | ✅ | ✅ | -| CDB FW Upgrade (`cdb_fw_upgrade.json`) | ✅ | - own reportable test | ✅ | +| CDB Firmware Upgrade (`cdb_firmware_upgrade.json`) | ✅ | - own reportable test | ✅ | | DOM (`dom.json`) | ✅ | ✅ | ✅ | | VDM (`vdm.json`) | ✅ | ✅ | ✅ | | PM (`pm.json`) | ✅ | ✅ | ✅ | | Port Config (`port_config.json`) | - CONFIG_DB only | - CONFIG_DB only | - CONFIG_DB only | -A "-" entry means the category intentionally does not consume that gate; the trailing note explains why. EEPROM and CDB FW Upgrade skip the gates whose semantics they own as reportable tests (so a gold-FW mismatch surfaces as a CDB FW Upgrade test failure, not a session-wide skip). +A "-" entry means the category intentionally does not consume that gate; the trailing note explains why. EEPROM and CDB Firmware Upgrade skip the gates whose semantics they own as reportable tests (so a gold-FW mismatch surfaces as a CDB Firmware Upgrade test failure, not a session-wide skip). #### Common Per-Test Health Checks @@ -622,7 +622,7 @@ attributes/ │ └── ... # same shape as eeprom/ ├── physical_oir/ ├── remote_reseat/ -├── cdb_fw_upgrade/ +├── cdb_firmware_upgrade/ ├── dom/ ├── vdm/ ├── pm/ @@ -654,7 +654,7 @@ attributes/ - `system/` (System tests) - `physical_oir/` (Physical OIR) - `remote_reseat/` (Remote reseat) -- `cdb_fw_upgrade/` (CDB FW upgrade tests - see also [Transceiver Firmware Info File](#transceiver-firmware-info-file)) +- `cdb_firmware_upgrade/` (CDB firmware upgrade tests - see also [Transceiver Firmware Info File](#transceiver-firmware-info-file)) - `dom/` (DOM) - `vdm/` (VDM) - `pm/` (PM) diff --git a/tests/acl/test_acl.py b/tests/acl/test_acl.py index 09513cc12a0..a907a21aba0 100644 --- a/tests/acl/test_acl.py +++ b/tests/acl/test_acl.py @@ -509,6 +509,16 @@ def setup(duthosts, ptfhost, rand_selected_dut, rand_selected_front_end_dut, ran # In multi-asic we need config both in host and namespace. if v['namespace']: acl_table_ports[''].append(k) + # Add upstream ports not covered by any PortChannel + pc_members = set() + for pc in port_channels.values(): + pc_members.update(pc.get('members', [])) + for namespace, port in list(upstream_ports.items()): + non_pc_ports = [p for p in port if p not in pc_members] + acl_table_ports[namespace] += non_pc_ports + # In multi-asic we need config both in host and namespace. + if namespace: + acl_table_ports[''] += non_pc_ports elif topo == "t2": acl_table_ports = t2_info['acl_table_ports'] elif topo == "lt2": diff --git a/tests/acl/test_stress_acl.py b/tests/acl/test_stress_acl.py index c05b99dc441..7b691d053f5 100644 --- a/tests/acl/test_stress_acl.py +++ b/tests/acl/test_stress_acl.py @@ -40,6 +40,7 @@ ACL_RULE_NUMS = 10 DEFAULT_MAX_ACL_ENTRIES = 200 +PTF_TIMEOUT = 10 # key: platform name # value: max number of ACL entries supported by the platform @@ -200,7 +201,8 @@ def prepare_test_port(rand_selected_dut, tbinfo): (topo == "t0" and ("T1" in neighbor["name"] or "PT0" in neighbor["name"])) or \ (topo == "m0" and "M1" in neighbor["name"]) or (topo == "mx" and "M0" in neighbor["name"]) or \ (topo == "m1" and ("MA" in neighbor["name"] or "MB" in neighbor["name"])) or \ - (topo_name in ("t1-isolated-d32", "t1-isolated-d128") and "T0" in neighbor["name"]): + (topo_name in ("t1-isolated-d32", "t1-isolated-d128", "t1-isolated-d32u1s2") + and "T0" in neighbor["name"]): upstream_ports[neighbor['namespace']].append(interface) upstream_port_ids.append(port_id) ipv4_addr = [bgp_neighbor['addr'] for bgp_neighbor in mg_facts['minigraph_bgp'] @@ -247,7 +249,7 @@ def verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, ptfadapter.dataplane.flush() testutils.send(test=ptfadapter, port_id=ptf_src_port, pkt=pkt) if verity_status == "forward" or acl_id == del_rule_id: - testutils.verify_packet_any_port(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) + testutils.verify_packet_any_port(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports, timeout=PTF_TIMEOUT) elif verity_status == "drop" and acl_id != del_rule_id: testutils.verify_no_packet_any(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) diff --git a/tests/autorestart/test_container_autorestart.py b/tests/autorestart/test_container_autorestart.py index 0d35d852351..7faa33970f8 100644 --- a/tests/autorestart/test_container_autorestart.py +++ b/tests/autorestart/test_container_autorestart.py @@ -3,9 +3,13 @@ """ import logging import re +import signal +import time from collections import defaultdict +from contextlib import contextmanager import pytest +from _pytest.outcomes import OutcomeException from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert @@ -29,8 +33,88 @@ POST_CHECK_INTERVAL_SECS = 1 POST_CHECK_THRESHOLD_SECS = 360 POST_CHECK_THRESHOLD_SECS_T2 = 600 +POST_CHECK_THRESHOLD_SECS_TH6_128 = 600 PROGRAM_STATUS = "RUNNING" +# Per-container watchdog budget for a single parametrized autorestart case. +# Each test_containers_autorestart[] exercises exactly one container; the +# slowest legitimate container (telemetry) finishes in ~22 min, so 40 min gives a +# comfortable ~1.8x margin for healthy runs. A container that hangs during +# stop/restart or post-check is interrupted here and fails with a junit entry, +# instead of blocking until the framework's module-level timeout (~155 min) kills +# the run and discards all results as phantom "no xml file" failures. +SINGLE_CONTAINER_TEST_TIMEOUT_SECS = 2400 + +# Upper bound for the best-effort config_reload that recovers the DUT after a +# per-container hang. A healthy `config_reload(safe_reload, wait_for_bgp)` -- even +# on a slow/modular topology -- completes well within this; if recovery itself +# exceeds it, something is badly wrong and we just log and fail the case anyway. +RECOVERY_RELOAD_TIMEOUT_SECS = 600 + + +class ContainerAutorestartTimeout(BaseException): + """Raised when a single container's autorestart sub-test exceeds its watchdog budget. + + Inherits from BaseException (not Exception) on purpose. The autorestart sub-test + polls DUT state through common.utilities.wait_until, whose retry loop catches + ``except Exception`` and treats any error as "condition not met yet, keep polling". + A plain Exception raised from the SIGALRM handler would be swallowed there, the + one-shot alarm consumed, and the case would then run unbounded -- defeating the + watchdog. As a BaseException (like KeyboardInterrupt/SystemExit and pytest's own + OutcomeException) it bypasses those ``except Exception`` loops and propagates up to + the per-case handler in test_containers_autorestart. + """ + + +@contextmanager +def single_container_timeout(container_name, timeout_secs=SINGLE_CONTAINER_TEST_TIMEOUT_SECS): + """Interrupt a single container's autorestart sub-test if it hangs. + + Uses SIGALRM so a blocked SSH/docker call is actually interrupted -- a thread + based timer cannot break out of a blocking C-level read. The signal interrupts + the blocking syscall and, because the handler raises, the exception propagates + out instead of the call being retried. Only effective on the main thread on + POSIX; on platforms without SIGALRM, or when not running on the main thread, it + is a no-op and execution falls back to the framework's module-level timeout. + """ + if not hasattr(signal, "SIGALRM"): + yield + return + + def _on_timeout(signum, frame): + raise ContainerAutorestartTimeout( + "Autorestart test for container '{}' exceeded {} seconds and was interrupted. " + "The container most likely hung during stop/restart or post-check.".format( + container_name, timeout_secs) + ) + + try: + previous_handler = signal.signal(signal.SIGALRM, _on_timeout) + except ValueError: + # signal handlers can only be installed on the main thread. + logger.warning("single_container_timeout disabled for '%s': not on main thread", container_name) + yield + return + + # signal.alarm() returns the seconds left on any previously scheduled alarm + # (0 if none). Save it so the watchdog restores -- rather than silently cancels + # -- a SIGALRM that pytest, the framework, or another fixture may have armed, + # instead of clobbering process-global timer state. Arming inside the try also + # keeps the handler restoration in finally even if signal.alarm() ever raised. + previous_alarm_remaining = 0 + armed_at = time.monotonic() + try: + previous_alarm_remaining = signal.alarm(timeout_secs) + yield + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_alarm_remaining > 0: + # Re-arm the pre-existing alarm, debited by the time we held it + # (at least 1s -- alarm(0) would cancel it instead). + elapsed = int(time.monotonic() - armed_at) + signal.alarm(max(1, previous_alarm_remaining - elapsed)) + @pytest.fixture(autouse=True, scope='module') def config_reload_after_tests(duthosts, selected_rand_one_per_hwsku_hostname, tbinfo): @@ -461,8 +545,11 @@ def postcheck_critical_processes_status(duthost, feature_autorestart_states, up_ if is_hiting_start_limit(duthost, feature_name): clear_failed_flag_and_restart(duthost, feature_name, feature_name) - post_check_threshold = POST_CHECK_THRESHOLD_SECS_T2 if duthost.get_facts().get("modular_chassis") \ - else POST_CHECK_THRESHOLD_SECS + post_check_threshold = POST_CHECK_THRESHOLD_SECS + if duthost.get_facts().get("modular_chassis"): + post_check_threshold = POST_CHECK_THRESHOLD_SECS_T2 + if duthost.sonichost.facts['platform'] == 'x86_64-nokia_ixr7220_h6_128-r0': + post_check_threshold = POST_CHECK_THRESHOLD_SECS_TH6_128 critical_proceses = wait_until( post_check_threshold, POST_CHECK_INTERVAL_SECS, 0, @@ -628,4 +715,32 @@ def test_containers_autorestart(duthosts, enum_rand_one_per_hwsku_hostname, enum asic = duthost.asic_instance(enum_rand_one_asic_index) service_name = asic.get_service_name(enum_dut_feature) container_name = asic.get_docker_name(enum_dut_feature) - run_test_on_single_container(duthost, container_name, service_name, tbinfo) + try: + with single_container_timeout(container_name): + run_test_on_single_container(duthost, container_name, service_name, tbinfo) + except ContainerAutorestartTimeout as timeout_err: + # Recover the DUT before failing so the remaining per-container cases in this + # module are not poisoned by a half-restarted container, then surface the hang + # as a normal failure (with junit) instead of a phantom run. The recovery is + # itself bounded so it cannot hang the module either. + logger.error(str(timeout_err)) + try: + with single_container_timeout(container_name, timeout_secs=RECOVERY_RELOAD_TIMEOUT_SECS): + config_reload(duthost, safe_reload=True, wait_for_bgp=True) + enable_autorestart(duthost) + except (KeyboardInterrupt, SystemExit): + raise + except ContainerAutorestartTimeout as recovery_timeout_err: + # Recovery itself exceeded RECOVERY_RELOAD_TIMEOUT_SECS. ContainerAutorestartTimeout + # is a BaseException (so wait_until cannot swallow it) and is therefore not caught by + # the (Exception, OutcomeException) handler below; catch it explicitly here, log it, + # and still fail with the original hang reason rather than letting it propagate raw. + logger.error("Recovery after hang on container '%s' itself timed out: %s", + container_name, recovery_timeout_err) + except (Exception, OutcomeException) as recovery_err: + # Recovery itself failed -- e.g. a config_reload assertion such as BGP-not-converged, + # which pytest raises as Failed (an OutcomeException, not a plain Exception). Log it + # with the traceback but still fail with the original hang reason so triage sees which + # container hung rather than a masked recovery error. + logger.exception("Recovery after hang on container '%s' failed: %s", container_name, recovery_err) + pytest.fail(str(timeout_err)) diff --git a/tests/bgp/bgp_helpers.py b/tests/bgp/bgp_helpers.py index 84b4dd0035e..7b0e061c3e4 100644 --- a/tests/bgp/bgp_helpers.py +++ b/tests/bgp/bgp_helpers.py @@ -21,6 +21,7 @@ from tests.common.utilities import wait_until from tests.common.utilities import is_ipv6_only_topology from tests.common.utilities import testbed_is_multi_vrf +from tests.common.utilities import get_neighbor_exabgp_vm_offset from tests.bgp.traffic_checker import get_traffic_shift_state from tests.bgp.constants import TS_NORMAL from tests.common.devices.eos import EosHost @@ -303,7 +304,13 @@ def bgp_allow_list_setup(tbinfo, nbrhosts, duthosts, rand_one_dut_hostname): if upstream_neighbors: other_neighbors += upstream_neighbors[0:2] - downstream_offset = tbinfo['topo']['properties']['topology']['VMs'][downstream]['vm_offset'] + # On converged (multi-VRF) topologies ``VMs[downstream]['vm_offset']`` is the + # collapsed *prime* offset, which maps to a different neighbor's exabgp + # instance. The per-neighbor exabgp instances are keyed by the ORIGINAL + # offset, so use that (via get_neighbor_exabgp_vm_offset) to post the + # announce to the intended downstream's exabgp. On stock topologies this + # returns the neighbor's own vm_offset, so behavior is unchanged. + downstream_offset = get_neighbor_exabgp_vm_offset(nbrhosts, tbinfo, downstream) downstream_exabgp_port = EXABGP_BASE_PORT + downstream_offset downstream_exabgp_port_v6 = EXABGP_BASE_PORT_V6 + downstream_offset diff --git a/tests/bgp/conftest.py b/tests/bgp/conftest.py index ba487392588..4b625751944 100644 --- a/tests/bgp/conftest.py +++ b/tests/bgp/conftest.py @@ -40,7 +40,7 @@ def check_results(results): """ failed_results = {} for node_name, node_results in list(results.items()): - failed_node_results = [res for res in node_results if res['failed']] + failed_node_results = [res for res in node_results if res.get('failed', False)] if len(failed_node_results) > 0: failed_results[node_name] = failed_node_results if failed_results: diff --git a/tests/bgp/test_4-byte_asn_community.py b/tests/bgp/test_4-byte_asn_community.py index a2d97a1fdf5..1e5b28405b6 100644 --- a/tests/bgp/test_4-byte_asn_community.py +++ b/tests/bgp/test_4-byte_asn_community.py @@ -193,6 +193,7 @@ def setup_ceos(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand cli_options = '' dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + confed_asn = duthost.get_bgp_confed_asn() neighbors = dict() bgp_facts = duthost.bgp_facts(instance_id=asic_index)['ansible_facts'] @@ -221,8 +222,13 @@ def setup_ceos(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand neigh_cli_options = '' - dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][dut_asn][0] - dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][dut_asn][1] + peer_in_bgp_confed = tbinfo['topo']['properties']['configuration'][neigh]['bgp'].get('peer_in_bgp_confed', False) + if peer_in_bgp_confed: + asn = int(confed_asn) + else: + asn = int(dut_asn) + dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][asn][0] + dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][asn][1] dut_ip_bgp_sum = duthost.shell('show ip bgp summary')['stdout'] @@ -291,6 +297,7 @@ def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand_one_ cli_options = '' dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + confed_asn = duthost.get_bgp_confed_asn() neigh = duthost.shell("show lldp table")['stdout'].split("\n")[3].split()[1] logger.debug("Neighbor is: {}".format(neigh)) @@ -318,8 +325,13 @@ def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand_one_ else: neigh_cli_options = '' - dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][dut_asn][0] - dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][dut_asn][1] + peer_in_bgp_confed = tbinfo['topo']['properties']['configuration'][neigh]['bgp'].get('peer_in_bgp_confed', False) + if peer_in_bgp_confed: + asn = int(confed_asn) + else: + asn = int(dut_asn) + dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][asn][0] + dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh]['bgp']['peers'][asn][1] dut_ip_bgp_sum = duthost.shell('show ip bgp summary')['stdout'] neigh_ip_bgp_sum = nbrhosts[neigh]["host"].shell('show ip bgp summary')['stdout'] diff --git a/tests/bgp/test_bgp_authentication.py b/tests/bgp/test_bgp_authentication.py index 65b4f915d68..648f3d6bbdd 100644 --- a/tests/bgp/test_bgp_authentication.py +++ b/tests/bgp/test_bgp_authentication.py @@ -26,15 +26,16 @@ @pytest.fixture(scope='module') def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, request): neighbor_type = request.config.getoption("neighbor_type") - if neighbor_type not in ["sonic", "eos"]: + if neighbor_type not in ["sonic", "csonic", "eos"]: pytest.skip("Unsupported neighbor type: {}".format(neighbor_type)) is_sonic_neigh = True - if neighbor_type != "sonic": + if neighbor_type not in ("sonic", "csonic"): is_sonic_neigh = False duthost = duthosts[enum_frontend_dut_hostname] dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + confed_asn = duthost.get_bgp_confed_asn() lldp_table = duthost.shell("show lldp table")['stdout'].split("\n")[3].split() tor1 = lldp_table[1] @@ -83,8 +84,13 @@ def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, request): else: neigh_eos_bgp_parents = eos_bgp_neighbor_config_parents(tbinfo, nbrhosts, tor1, neigh_asn) - dut_ip_v4 = tbinfo['topo']['properties']['configuration'][tor1]['bgp']['peers'][dut_asn][0] - dut_ip_v6 = tbinfo['topo']['properties']['configuration'][tor1]['bgp']['peers'][dut_asn][1] + peer_in_bgp_confed = tbinfo['topo']['properties']['configuration'][tor1]['bgp'].get('peer_in_bgp_confed', False) + if peer_in_bgp_confed: + asn = int(confed_asn) + else: + asn = int(dut_asn) + dut_ip_v4 = tbinfo['topo']['properties']['configuration'][tor1]['bgp']['peers'][asn][0] + dut_ip_v6 = tbinfo['topo']['properties']['configuration'][tor1]['bgp']['peers'][asn][1] logger.info("default namespace {}".format(DEFAULT_NAMESPACE)) diff --git a/tests/bgp/test_bgp_bbr.py b/tests/bgp/test_bgp_bbr.py index 32b6ccc1287..6c315a752e8 100644 --- a/tests/bgp/test_bgp_bbr.py +++ b/tests/bgp/test_bgp_bbr.py @@ -19,6 +19,7 @@ from tests.common.helpers.parallel import reset_ansible_local_tmp from tests.common.helpers.parallel import parallel_run from tests.common.utilities import wait_until, delete_running_config +from tests.common.utilities import get_neighbor_exabgp_vm_offset from tests.common.gu_utils import apply_patch, expect_op_success from tests.common.gu_utils import generate_tmpfile, delete_tmpfile from tests.common.gu_utils import format_json_patch_for_multiasic @@ -179,7 +180,7 @@ def setup(duthosts, rand_one_dut_hostname, tbinfo, nbrhosts): other_vms.append(neigh['name']) # Announce route to one of the T0 VM - tor1_offset = tbinfo['topo']['properties']['topology']['VMs'][tor1]['vm_offset'] + tor1_offset = get_neighbor_exabgp_vm_offset(nbrhosts, tbinfo, tor1) tor1_exabgp_port = EXABGP_BASE_PORT + tor1_offset tor1_exabgp_port_v6 = EXABGP_BASE_PORT_V6 + tor1_offset @@ -408,11 +409,11 @@ def _check_route(): bgp_neighbors = json.loads(duthost.shell("sonic-cfggen -d --var-json 'BGP_NEIGHBOR'")['stdout']) # check tor1 - pytest_assert(wait_until(5, 1, 0, check_tor1, nbrhosts, setup, route), 'tor1 check failed') + pytest_assert(wait_until(60, 5, 0, check_tor1, nbrhosts, setup, route), 'tor1 check failed') # check DUT - pytest_assert(wait_until(30, 1, 0, check_dut, duthost, other_vms, bgp_neighbors, - setup, route, accepted=accepted), 'DUT check failed') + pytest_assert(wait_until(120, 5, 0, check_dut, duthost, list(dict.fromkeys(other_vms)), + bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed') results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120, concurrent_tasks=6) diff --git a/tests/bgp/test_bgp_bbr_default_state.py b/tests/bgp/test_bgp_bbr_default_state.py index 77afb49969c..a14aba3a01c 100644 --- a/tests/bgp/test_bgp_bbr_default_state.py +++ b/tests/bgp/test_bgp_bbr_default_state.py @@ -13,6 +13,7 @@ from tests.common.gu_utils import generate_tmpfile, delete_tmpfile from tests.common.gu_utils import format_json_patch_for_multiasic from tests.common.config_reload import config_reload +from bgp_bbr_helpers import get_bbr_default_state pytestmark = [ @@ -97,6 +98,33 @@ def disable_bbr(duthost, namespace): config_bbr_by_gcu(duthost, "disabled") +def get_bbr_status_from_config_db(duthost, namespace): + namespace_prefix = '-n ' + namespace if namespace else '' + return duthost.shell( + 'sonic-db-cli {} CONFIG_DB HGET "BGP_BBR|all" "status"'.format(namespace_prefix) + )['stdout'].strip().strip('"') + + +def get_external_allowas_lines(duthost, namespace): + bgp_cmd = 'vtysh -c "show running-configuration bgp"' + cmd = duthost.get_vtysh_cmd_for_namespace(bgp_cmd, namespace) + output = duthost.shell(cmd, module_ignore_errors=True)['stdout'] + return [ + line.strip() for line in output.splitlines() + if 'allowas' in line and 'INTERNAL_PEER' not in line + ] + + +def verify_bbr_disabled(duthost, namespace): + bbr_status = get_bbr_status_from_config_db(duthost, namespace) + pytest_assert(bbr_status == 'disabled', + "BGP_BBR status is '{}', expected 'disabled'".format(bbr_status)) + + external_allowas = get_external_allowas_lines(duthost, namespace) + pytest_assert(not external_allowas, + "BBR allowas-in found on external peers: {}".format(external_allowas)) + + @pytest.fixture def config_bbr_disabled(duthosts, setup, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] @@ -109,17 +137,27 @@ def setup(duthosts, rand_one_dut_hostname, tbinfo, nbrhosts): constants_stat = duthost.stat(path=CONSTANTS_FILE) if not constants_stat['stat']['exists']: pytest.skip('No file {} on DUT, BBR is not supported') - bbr_default_state = 'disabled' + + bbr_supported, bbr_default_state = get_bbr_default_state(duthost) + if not bbr_supported: + pytest.skip('BGP BBR is not supported') + if bbr_default_state != 'disabled': + pytest.skip('Test only applies when constants.yml BBR default is disabled') + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) tor_neighbors = natsorted([neighbor for neighbor in list(nbrhosts.keys()) if neighbor.endswith('T0')]) + pytest_assert(tor_neighbors, 'No T0 neighbor found in topology') tor1 = tor_neighbors[0] + tor1_namespace = DEFAULT_NAMESPACE - for dut_port, neigh in list(mg_facts['minigraph_neighbors'].items()): + for _, neigh in list(mg_facts['minigraph_neighbors'].items()): if tor1 == neigh['name']: - tor1_namespace = neigh['namespace'] + tor1_namespace = neigh.get('namespace', DEFAULT_NAMESPACE) break + setup_info = { 'bbr_default_state': bbr_default_state, + 'tor1': tor1, 'tor1_namespace': tor1_namespace, } if not setup_info['tor1_namespace']: @@ -134,5 +172,4 @@ def test_bbr_disabled_constants_yml_default(duthosts, rand_one_dut_hostname, set duthost = duthosts[rand_one_dut_hostname] duthost.shell("sudo config save -y") config_reload(duthost, safe_reload=True) - is_bbr_enabled = duthost.shell("show runningconfiguration bgp | grep allowas", module_ignore_errors=True)['stdout'] - pytest_assert(is_bbr_enabled == "", "BBR is not disabled when it should be.") + verify_bbr_disabled(duthost, setup['tor1_namespace']) diff --git a/tests/bgp/test_bgp_multipath_relax.py b/tests/bgp/test_bgp_multipath_relax.py index 7725439ca3f..7abfe97937f 100644 --- a/tests/bgp/test_bgp_multipath_relax.py +++ b/tests/bgp/test_bgp_multipath_relax.py @@ -22,7 +22,14 @@ def get_t0_neigh(tbinfo, topo_config): get all t0 router names which has vips defined """ dut_t0_neigh = [] - for vm in list(tbinfo['topo']['properties']['topology']['VMs'].keys()): + # Enumerate from the topology ``configuration`` (which lists every logical + # neighbor) rather than ``topology.VMs``. On converged (multi-VRF) topologies + # ``topology.VMs`` only contains the collapsed prime devices (one per role), + # so merged sub-peers (e.g. ARISTA03T0) that also advertise the vips prefix + # are absent there, leaving fewer than 2 multipath sources. The per-neighbor + # ``configuration`` is preserved intact by the converger, so it carries every + # logical T0 and its ``vips`` on both stock and converged topologies. + for vm in list(topo_config.keys()): if 'T0' in vm: if 'vips' in topo_config[vm]: dut_t0_neigh.append(vm) diff --git a/tests/bgp/test_bgp_peer_shutdown.py b/tests/bgp/test_bgp_peer_shutdown.py index eca2590e4fd..40e76332fc7 100644 --- a/tests/bgp/test_bgp_peer_shutdown.py +++ b/tests/bgp/test_bgp_peer_shutdown.py @@ -59,6 +59,11 @@ def common_setup_teardown( if dut_type in ["ToRRouter", "SpineRouter", "BackEndToRRouter", "LowerSpineRouter"]: neigh_type = "LeafRouter" + elif dut_type == "UpperSpineRouter" and confed_asn is not None: + # On confederation-based UT2 topologies the UpperSpineRouter peers with + # AZNGHub neighbors using the confederation ASN, not the per-DUT ASN. + neigh_type = "AZNGHub" + dut_asn = int(confed_asn) elif dut_type in ["UpperSpineRouter", "FabricSpineRouter"]: neigh_type = "LowerSpineRouter" if dut_type == "FabricSpineRouter" and confed_asn is not None: @@ -72,7 +77,6 @@ def common_setup_teardown( neigh_type = "LowerRegionalHub" if confed_asn is not None: use_vtysh = True - else: neigh_type = "ToRRouter" logging.info( diff --git a/tests/bgp/test_bgp_route_neigh_learning.py b/tests/bgp/test_bgp_route_neigh_learning.py index bcf0cb63f9a..9652f78243f 100644 --- a/tests/bgp/test_bgp_route_neigh_learning.py +++ b/tests/bgp/test_bgp_route_neigh_learning.py @@ -4,6 +4,7 @@ from tests.common.helpers.assertions import pytest_assert as py_assert from tests.common.devices.eos import EosHost from tests.common.devices.sonic import SonicHost +from tests.common.devices.csonic import CsonicHost from tests.common.utilities import wait_until pytestmark = [ pytest.mark.topology('t0'), @@ -118,7 +119,7 @@ def fixture_setUp(nbrhosts, duthosts, enum_frontend_dut_hostname, ip_version): # remove the route in the neighbor T1 eos device if isinstance(nbrhost, EosHost): rm_route_from_nbr(data, name, prefix, mask, afi_cfg["afi_cmd_eos"]) - elif isinstance(nbrhost, SonicHost): + elif isinstance(nbrhost, (SonicHost, CsonicHost)): cmd = "sudo vtysh -c 'configure terminal' " \ f"-c 'router bgp {bgp_as_num}' " \ f"-c \"{afi_cfg['afi_cmd_sonic']}\" " \ @@ -156,7 +157,7 @@ def run_bgp_neighbor_route_learning(duthosts, enum_frontend_dut_hostname, data): # add a route in the neighbor T1 eos device if isinstance(nbrhost, EosHost): add_route_to_nbr(data, name, prefix, mask, afi_cfg["afi_cmd_eos"], afi_cfg["loopback_cmd_eos"]) - elif isinstance(nbrhost, SonicHost): + elif isinstance(nbrhost, (SonicHost, CsonicHost)): # Create and configure loopback interface cmd = f"sudo config interface ip add Loopback1 {prefix}/{mask}" result = nbrhost.shell(cmd) @@ -174,7 +175,7 @@ def run_bgp_neighbor_route_learning(duthosts, enum_frontend_dut_hostname, data): duthost = duthosts[enum_frontend_dut_hostname] Logger.info("checking DUT for route %s", prefix) - is_route_propagated = wait_until(10, 2, 0, lambda: _check_route_propagation(duthost, data)) + is_route_propagated = wait_until(60, 5, 0, lambda: _check_route_propagation(duthost, data)) py_assert(is_route_propagated, "Route did not propagate to the DUT") diff --git a/tests/bgp/test_bgp_router_id.py b/tests/bgp/test_bgp_router_id.py index eabc5d91872..57b3b1955ba 100644 --- a/tests/bgp/test_bgp_router_id.py +++ b/tests/bgp/test_bgp_router_id.py @@ -20,7 +20,7 @@ def verify_bgp_peer(neighbor_type, nbrhost, localip, expected_bgp_router_id, is_v6_topo, vrf="default"): - if neighbor_type == "sonic": + if neighbor_type in ("sonic", "csonic"): if is_v6_topo: cmd = "show ipv6 bgp neighbors {}".format(localip) else: @@ -59,7 +59,7 @@ def verify_bgp(enum_asic_index, duthost, expected_bgp_router_id, neighbor_type, run_bgp_facts(duthost, enum_asic_index) # Verify from peer device side to check - if neighbor_type not in ["sonic", "eos"]: + if neighbor_type not in ["sonic", "csonic", "eos"]: logger.warning("Unsupport neighbor type for neighbor bgp check: {}".format(neighbor_type)) local_ip_map = {} cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] diff --git a/tests/bgp/test_bgp_suppress_fib.py b/tests/bgp/test_bgp_suppress_fib.py index 483d57faffb..109e5b7d1eb 100644 --- a/tests/bgp/test_bgp_suppress_fib.py +++ b/tests/bgp/test_bgp_suppress_fib.py @@ -822,7 +822,7 @@ def param_reboot(request, duthost, localhost, loganalyzer): logger.info("Randomly choose {} from reload, cold, warm, fast".format(reboot_type)) if reboot_type == "reload": - config_reload(duthost, safe_reload=True, ignore_loganalyzer=loganalyzer) + config_reload(duthost, safe_reload=True, ignore_loganalyzer=loganalyzer, wait_for_bgp=True) wait_until(120, 10, 0, check_interface_status, duthost) # Wait for BGP sessions to re-establish, consistent with do_and_wait_reboot() bgp_neighbors = duthost.get_bgp_neighbors_per_asic(state="all") diff --git a/tests/bgp/test_bgp_update_replication.py b/tests/bgp/test_bgp_update_replication.py index 62f2f0956b5..8b705246cf6 100644 --- a/tests/bgp/test_bgp_update_replication.py +++ b/tests/bgp/test_bgp_update_replication.py @@ -19,6 +19,8 @@ # Fixture params PEER_COUNT = 16 WAIT_TIMEOUT = 120 +STATS_COLLECT_TIMEOUT = 30 +STATS_COLLECT_INTERVAL = 5 pytestmark = [ pytest.mark.topology('t0', 't1', 't2', 'lrh', 'urh', 'lt2', 'ft2'), @@ -50,6 +52,31 @@ def generate_routes(num_routes, nexthop, is_ipv6=False): } +def _read_parsed_stats(dut, cmd, template_path): + ''' + Run a stats command on the DUT and TextFSM-parse its output, retrying while the DUT + returns empty/unparseable output. Under the heavy route churn this test drives, the DUT + management plane can momentarily return empty output for "show ip bgp summary", which + would otherwise crash the caller with an IndexError when indexing the empty parse. + ''' + parsed = [] + + def _parsed_ready(): + nonlocal parsed + output = dut.shell(cmd, module_ignore_errors=True)['stdout'] + with open(template_path) as template: + parsed = textfsm.TextFSM(template).ParseTextToDicts(output) + if not parsed: + logger.warning("Empty parse for '%s', retrying; output=%r", cmd, output) + return bool(parsed) + + pytest_assert( + wait_until(STATS_COLLECT_TIMEOUT, STATS_COLLECT_INTERVAL, 0, _parsed_ready), + f"DUT did not return parseable output for '{cmd}' within {STATS_COLLECT_TIMEOUT} sec" + ) + return parsed + + def measure_stats(dut, is_ipv6=False): ''' Validates that the provided DUT is responsive during test, and that device stats do not @@ -64,11 +91,11 @@ def measure_stats(dut, is_ipv6=False): time_before_cmd = time.process_time() - proc_cpu = dut.shell("show processes cpu | head -n 10", module_ignore_errors=True)['stdout'] + parsed_proc = _read_parsed_stats(dut, "show processes cpu | head -n 10", PROC_TEMPLATE) time_first_cmd = time.process_time() bgp_cmd = f"show ip{'v6' if is_ipv6 else ''} bgp summary | grep memory" - bgp_sum = dut.shell(bgp_cmd, module_ignore_errors=True)['stdout'] + parsed_bgp_sum = _read_parsed_stats(dut, bgp_cmd, BGP_SUM_TEMPLATE) time_second_cmd = time.process_time() num_cores = dut.shell('cat /proc/cpuinfo | grep "cpu cores" | uniq', module_ignore_errors=True)['stdout'] @@ -87,14 +114,6 @@ def measure_stats(dut, is_ipv6=False): f"SSH session took longer than average of {responsive_threshold} sec to respond" ) - with open(PROC_TEMPLATE) as template: - fsm = textfsm.TextFSM(template) - parsed_proc = fsm.ParseTextToDicts(proc_cpu) - - with open(BGP_SUM_TEMPLATE) as template: - fsm = textfsm.TextFSM(template) - parsed_bgp_sum = fsm.ParseTextToDicts(bgp_sum) - stats: dict[str, Any] = {"timestamp": datetime.datetime.now().time()} stats.update(parsed_proc[0]) stats.update(parsed_bgp_sum[0]) @@ -159,6 +178,11 @@ def setup_bgp_peers( dut_type = mg_facts["minigraph_devices"][duthost.hostname]["type"] if dut_type in ["ToRRouter", "SpineRouter", "BackEndToRRouter", "LowerSpineRouter"]: neigh_type = "LeafRouter" + elif dut_type == "UpperSpineRouter" and confed_asn is not None: + # On confederation-based UT2 topologies the UpperSpineRouter peers with + # AZNGHub neighbors using the confederation ASN, not the per-DUT ASN. + neigh_type = "AZNGHub" + dut_asn = int(confed_asn) elif dut_type in ["UpperSpineRouter", "FabricSpineRouter"]: neigh_type = "LowerSpineRouter" if dut_type == "FabricSpineRouter" and confed_asn is not None: diff --git a/tests/bgp/test_bgp_update_timer.py b/tests/bgp/test_bgp_update_timer.py index 5e9f5ea22ef..31f7b601f35 100644 --- a/tests/bgp/test_bgp_update_timer.py +++ b/tests/bgp/test_bgp_update_timer.py @@ -99,10 +99,14 @@ def _apply_outbound_route_filter(duthost, dut_asn, neighbor_ips, is_v6, namespac cmd = "vtysh {} {}".format(ns_option, " ".join("-c '{}'".format(c) for c in vtysh_cmds)) duthost.shell(cmd) - # Soft-reset outbound so the filter takes effect immediately + # Soft-reset outbound so the filter takes effect immediately. + # Note: FRR vtysh syntax differs between v4 and v6: + # v4: clear ip bgp soft out + # v6: clear bgp ipv6 soft out (word order is 'bgp ipv6', not 'ipv6 bgp') + clear_af = "bgp ipv6" if is_v6 else "ip bgp" for ip in neighbor_ips: - duthost.shell("vtysh {} -c 'clear {} bgp {} soft out'".format( - ns_option, "ipv6" if is_v6 else "ip", ip + duthost.shell("vtysh {} -c 'clear {} {} soft out'".format( + ns_option, clear_af, ip )) @@ -181,6 +185,11 @@ def common_setup_teardown( if dut_type in ["ToRRouter", "SpineRouter", "BackEndToRRouter", "LowerSpineRouter"]: neigh_type = "LeafRouter" + elif dut_type == "UpperSpineRouter" and confed_asn is not None: + # On confederation-based UT2 topologies the UpperSpineRouter peers with + # AZNGHub neighbors using the confederation ASN, not the per-DUT ASN. + neigh_type = "AZNGHub" + dut_asn = int(confed_asn) elif dut_type in ["UpperSpineRouter", "FabricSpineRouter"]: neigh_type = "LowerSpineRouter" if dut_type == "FabricSpineRouter" and confed_asn is not None: diff --git a/tests/bgp/test_frr_set_src_route_map.py b/tests/bgp/test_frr_set_src_route_map.py new file mode 100644 index 00000000000..bf4462ee545 --- /dev/null +++ b/tests/bgp/test_frr_set_src_route_map.py @@ -0,0 +1,328 @@ +"""Ensure mgmtd FRR replays preserve default-route set-src even with large configs.""" + +import ipaddress +import logging +import pytest + +from tests.common import config_reload +from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.gcu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.utilities import wait_until + + +pytestmark = [ + pytest.mark.disable_loganalyzer, + pytest.mark.topology('t0', 't1', 't2') +] + + +BLOAT_PREFIX_LIST_COUNT = 512 +BLOAT_FRR_CONFIG_FILE = "/tmp/frr_set_src_bloat.conf" +ITERATION_LEVEL_MAP = { + 'debug': 1, + 'basic': 1, + 'confident': 2, + 'thorough': 3 +} +logger = logging.getLogger(__name__) + + +def _get_frontend_bgp_docker_names(duthost): + """Return bgp container names for frontend ASICs only.""" + if not duthost.is_multi_asic: + return ["bgp"] + return ["bgp{}".format(asic_id) for asic_id in duthost.get_frontend_asic_ids()] + + +def _mgmtd_running(duthost): + """Return True if mgmtd is running inside all frontend bgp containers.""" + for bgp_name in _get_frontend_bgp_docker_names(duthost): + if duthost.shell( + "docker exec {} pgrep -x mgmtd".format(bgp_name), module_ignore_errors=True + )['rc'] != 0: + return False + return True + + +def _get_asic_hosts(duthost): + """ + Provide a normalized list of ASIC hosts to iterate over. + """ + if duthost.is_multi_asic: + return [duthost.asic_instance(asic_index) for asic_index in duthost.get_frontend_asic_ids()] + return [duthost.asic_instance()] + + +def _extract_loopback_ips(asichost, duthost): + """ + Return the IPv4/IPv6 addresses configured on Loopback0 for the supplied ASIC. + """ + config_facts = asichost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + loopbacks = config_facts.get("LOOPBACK_INTERFACE", {}) + pytest_assert("Loopback0" in loopbacks, "Loopback0 missing from config facts") + + lo_ipv4 = None + lo_ipv6 = None + + for ip_str in loopbacks["Loopback0"]: + loop_ip = ipaddress.ip_interface(ip_str) + if loop_ip.version == 4: + lo_ipv4 = loop_ip + elif loop_ip.version == 6: + lo_ipv6 = loop_ip + + pytest_assert(lo_ipv4, "Failed to locate IPv4 Loopback0 address") + pytest_assert(lo_ipv6, "Failed to locate IPv6 Loopback0 address") + + return lo_ipv4, lo_ipv6 + + +def _verify_default_route_set_src(asichost, lo_ipv4, lo_ipv6): + """ + Ensure both IPv4 and IPv6 default routes carry the expected source address. + """ + ipv4_default = asichost.get_ip_route_info(ipaddress.ip_network("0.0.0.0/0")) + pytest_assert(ipv4_default.get("set_src"), "IPv4 default route missing set_src attribute") + pytest_assert( + ipv4_default["set_src"] == lo_ipv4.ip, + "IPv4 default route set_src {} does not match Loopback0 {}".format(ipv4_default['set_src'], lo_ipv4.ip) + ) + + ipv6_default = asichost.get_ip_route_info(ipaddress.ip_network("::/0")) + pytest_assert(ipv6_default.get("set_src"), "IPv6 default route missing set_src attribute") + pytest_assert( + ipv6_default["set_src"] == lo_ipv6.ip, + "IPv6 default route set_src {} does not match Loopback0 {}".format(ipv6_default['set_src'], lo_ipv6.ip) + ) + + +def _verify_route_maps_in_running_config(asichost, lo_ipv4, lo_ipv6): + """ + Confirm FRR running-config retains the route-maps that install the Loopback source-ip. + """ + running_cfg = asichost.run_vtysh("-c 'show running-config'")["stdout"] + pytest_assert("route-map RM_SET_SRC permit 10" in running_cfg, "RM_SET_SRC missing from running-config") + pytest_assert( + "set src {}".format(lo_ipv4.ip) in running_cfg, + "RM_SET_SRC missing the Loopback0 IPv4 address {}".format(lo_ipv4.ip) + ) + pytest_assert( + "ip protocol bgp route-map RM_SET_SRC" in running_cfg, + "RM_SET_SRC not bound to 'ip protocol bgp'" + ) + pytest_assert("route-map RM_SET_SRC6 permit 10" in running_cfg, "RM_SET_SRC6 missing from running-config") + pytest_assert( + "set src {}".format(lo_ipv6.ip) in running_cfg, + "RM_SET_SRC6 missing the Loopback0 IPv6 address {}".format(lo_ipv6.ip) + ) + pytest_assert( + "ipv6 protocol bgp route-map RM_SET_SRC6" in running_cfg, + "RM_SET_SRC6 not bound to 'ipv6 protocol bgp'" + ) + + +def _verify_set_src_all_asics(duthost): + """Run the default-route checks on every frontend ASIC.""" + for asichost in _get_asic_hosts(duthost): + lo_ipv4, lo_ipv6 = _extract_loopback_ips(asichost, duthost) + _verify_default_route_set_src(asichost, lo_ipv4, lo_ipv6) + _verify_route_maps_in_running_config(asichost, lo_ipv4, lo_ipv6) + + +def _check_set_src_all_asics(duthost): + """Non-asserting wrapper for use with wait_until.""" + try: + _verify_set_src_all_asics(duthost) + return True + except Exception as e: + logger.debug("_check_set_src_all_asics not yet passing: %s", e) + return False + + +def _generate_bloat_frr_config(count=BLOAT_PREFIX_LIST_COUNT): + """Generate FRR prefix-list + route-map lines to add extra config load.""" + assert count <= 65536, "count exceeds 65536; 198.18.x.y address space exhausted" + lines = [] + for i in range(count): + lines.append("ip prefix-list BLOAT_PL seq {} permit 198.18.{}.{}/32".format( + (i + 1) * 5, i // 256, i % 256 + )) + lines.append("route-map BLOAT_RM permit 10") + lines.append(" match ip address prefix-list BLOAT_PL") + return "\n".join(lines) + "\n" + + +def _inject_bloat_frr_config(duthost, count=BLOAT_PREFIX_LIST_COUNT): + """Inject bloat prefix-lists into FRR running config via vtysh -f. + + The config file is placed under /etc/frr/ inside the container rather than + /tmp/ because FRR 10.x daemons may run with PrivateTmp or separate mount + namespaces, making /tmp invisible to child processes spawned by vtysh -f. + /etc/frr/ is always shared across all FRR daemon processes. + """ + config_text = _generate_bloat_frr_config(count) + duthost.copy(content=config_text, dest=BLOAT_FRR_CONFIG_FILE) + for bgp_name in _get_frontend_bgp_docker_names(duthost): + duthost.shell("docker cp {} {}:/etc/frr/bloat.conf".format(BLOAT_FRR_CONFIG_FILE, bgp_name)) + duthost.shell("docker exec {} vtysh -f /etc/frr/bloat.conf".format(bgp_name)) + duthost.shell("docker exec {} rm -f /etc/frr/bloat.conf".format(bgp_name)) + duthost.shell("docker exec {} vtysh -c 'write memory'".format(bgp_name)) + logger.info("Injected %d bloat prefix-list entries into FRR config", count) + + +def _remove_bloat_frr_config(duthost): + """Remove injected bloat from FRR running-config and persist the clean state.""" + for bgp_name in _get_frontend_bgp_docker_names(duthost): + duthost.shell( + "docker exec {} vtysh -c 'configure terminal' " + "-c 'no route-map BLOAT_RM' " + "-c 'no ip prefix-list BLOAT_PL'".format(bgp_name), + module_ignore_errors=True + ) + duthost.shell( + "docker exec {} vtysh -c 'write memory'".format(bgp_name), + module_ignore_errors=True + ) + + +def _start_vtysh_race_loop(duthost): + """ + Start a background process on the DUT that continuously spawns competing + vtysh sessions. Returns (pid, pgid) so the caller can kill the process + group reliably even if the PID has exited by cleanup time. + + The loop must keep running across the config_reload so it catches the + window when the bgp container comes back and mgmtd starts replaying. + """ + bgp_names = _get_frontend_bgp_docker_names(duthost) + probe_cmds = "; ".join( + 'docker exec {c} vtysh -c "show version" &>/dev/null &'.format(c=c) + for c in bgp_names + ) + result = duthost.shell( + "nohup setsid bash -c 'while true; do for i in $(seq 1 5); do " + "{probes} done; wait; sleep 1; done' >/dev/null 2>&1 & " + "PID=$!; echo $PID $(ps -o pgid= -p $PID | tr -d ' ')".format(probes=probe_cmds), + module_ignore_errors=True + ) + parts = result["stdout"].strip().split() + pytest_assert( + len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit(), + "Failed to start vtysh race loop, got: '{}'".format(result["stdout"].strip()) + ) + pid, pgid = parts + logger.info("Started vtysh race loop (PID %s, PGID %s) targeting containers: %s", pid, pgid, bgp_names) + return pid, pgid + + +def _stop_vtysh_race_loop(duthost, pid, pgid): + """Kill the background vtysh race loop and its full process tree.""" + # Kill the entire process group (captured at start time for reliability), + # then the PID itself as a fallback, then mop up any orphaned docker exec + # vtysh processes. + duthost.shell( + "kill -- -{pgid} 2>/dev/null; kill {pid} 2>/dev/null".format(pid=pid, pgid=pgid), + module_ignore_errors=True + ) + # Belt and suspenders: kill any remaining docker exec vtysh processes + duthost.shell( + "pkill -f 'docker exec.*vtysh.*show version' 2>/dev/null", + module_ignore_errors=True + ) + + +def _race_loop_alive(duthost, pid): + """Return True if the race loop process is still running.""" + return duthost.shell( + "kill -0 {} 2>/dev/null".format(pid), module_ignore_errors=True + )['rc'] == 0 + + +def test_mgmtd_preserves_default_route_set_src( + dut_with_default_route, + get_function_completeness_level): + """ + Regress the mgmtd replay bug by forcing a config reload and ensuring FRR keeps the set src route-maps. + """ + duthost = dut_with_default_route + + pytest_require(_mgmtd_running(duthost), "Test requires mgmtd (FRR 10.x+)") + + normalized_level = get_function_completeness_level if get_function_completeness_level else 'debug' + iterations = ITERATION_LEVEL_MAP.get(normalized_level, ITERATION_LEVEL_MAP['debug']) + + logger.info("Running mgmtd set-src regression for %s iteration(s)", iterations) + + # Baseline: verify route-maps exist before reload + pytest_assert( + wait_until(60, 10, 0, _check_set_src_all_asics, duthost), + "Baseline check failed: RM_SET_SRC route-maps not present before reload" + ) + + for iteration in range(1, iterations + 1): + logger.info("Iteration %s/%s: issuing config reload with race amplification", iteration, iterations) + pid, pgid = _start_vtysh_race_loop(duthost) + try: + config_reload(duthost, safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) + if not _race_loop_alive(duthost, pid): + logger.warning( + "vtysh race loop (PID %s) died during config_reload — " + "this iteration did NOT exercise the mgmtd race condition", + pid + ) + finally: + _stop_vtysh_race_loop(duthost, pid, pgid) + + pytest_assert( + wait_until(120, 10, 0, _check_set_src_all_asics, duthost), + "RM_SET_SRC route-maps not restored within 120s after config reload" + ) + + +def test_mgmtd_preserves_default_route_set_src_with_large_config( + dut_with_default_route): + """ + Inject extra FRR config (prefix-lists) before reload to validate that + route-maps remain correct under additional FRR state. The bloat is + FRR-only and does not survive config_reload (container restart + regenerates frr.conf from CONFIG_DB). + """ + duthost = dut_with_default_route + + pytest_require(_mgmtd_running(duthost), "Test requires mgmtd (FRR 10.x+)") + + checkpoint_name = "set_src_bloat_cp" + + # Baseline: verify route-maps exist before reload + pytest_assert( + wait_until(60, 10, 0, _check_set_src_all_asics, duthost), + "Baseline check failed: RM_SET_SRC route-maps not present before reload" + ) + + create_checkpoint(duthost, checkpoint_name) + + try: + _inject_bloat_frr_config(duthost) + + # config save omitted — FRR-only state doesn't persist across reload + pid, pgid = _start_vtysh_race_loop(duthost) + try: + config_reload(duthost, safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) + if not _race_loop_alive(duthost, pid): + logger.warning( + "vtysh race loop (PID %s) died during config_reload — " + "this iteration did NOT exercise the mgmtd race condition", + pid + ) + finally: + _stop_vtysh_race_loop(duthost, pid, pgid) + + pytest_assert( + wait_until(120, 10, 0, _check_set_src_all_asics, duthost), + "RM_SET_SRC route-maps not restored within 120s after config reload" + ) + finally: + _remove_bloat_frr_config(duthost) + rollback_or_reload(duthost, checkpoint_name) + delete_checkpoint(duthost, checkpoint_name) + duthost.shell("rm -f {}".format(BLOAT_FRR_CONFIG_FILE), module_ignore_errors=True) diff --git a/tests/bgp/test_ipv6_bgp_scale.py b/tests/bgp/test_ipv6_bgp_scale.py index ebdb047084e..4572e764782 100644 --- a/tests/bgp/test_ipv6_bgp_scale.py +++ b/tests/bgp/test_ipv6_bgp_scale.py @@ -286,9 +286,30 @@ def _get_backplane_ports(): return {k for k, v in ptf.config.get('port_map', {}).items() if v == 'backplane'} +def wait_for_rx_quiescence(pdp, exp_mask, stable_secs=10, poll_interval=1, max_timeout=120): + """Wait until mask_rx_cnt stops incrementing for stable_secs seconds. + + After stopping the traffic thread, packets remain in transit through the + nn_agent pipeline (DUT -> veth -> AF_PACKET -> nn_agent -> nanomsg -> PTF). + This can take 12-17 seconds to drain. Polling until counters stabilize + avoids the false packet loss caused by a fixed-duration sleep. + """ + start = time.time() + prev_rx = sum(pdp.mask_rx_cnt[exp_mask].values()) + stable_since = start + while time.time() - stable_since < stable_secs: + if time.time() - start > max_timeout: + logger.warning("rx quiescence max timeout (%ds) exceeded; proceeding with current counters", max_timeout) + break + time.sleep(poll_interval) + curr_rx = sum(pdp.mask_rx_cnt[exp_mask].values()) + if curr_rx != prev_rx: + prev_rx = curr_rx + stable_since = time.time() + + def calculate_downtime(ptf_dp, end_time, start_time, masked_exp_pkt): - logger.warning("Waiting %d seconds for mask counters to be updated", MASK_COUNTER_WAIT_TIME) - time.sleep(MASK_COUNTER_WAIT_TIME) + wait_for_rx_quiescence(ptf_dp, masked_exp_pkt) backplane_ports = _get_backplane_ports() rx_total = sum( cnt for port_key, cnt in ptf_dp.mask_rx_cnt[masked_exp_pkt].items() @@ -342,6 +363,19 @@ def flush_counters(ptf_dp, masked_exp_pkt): ptf_dp.mask_rx_cnt[masked_exp_pkt], ptf_dp.mask_tx_cnt[masked_exp_pkt]) +def _send_with_retry(ptf_dataplane, device_num, port_num, pkt, max_retries=10, base_backoff=0.1): + """Send a single packet, retrying on nanomsg timeout (nn_agent CPU starvation).""" + for attempt in range(max_retries): + try: + ptf_dataplane.send(device_num, port_num, pkt) + return True + except Exception as e: + if "timed out" not in str(e).lower(): + raise + time.sleep(base_backoff * (2 ** min(attempt, 4))) + return False + + def send_packets( terminated, ptf_dataplane, @@ -355,14 +389,16 @@ def send_packets( pkts_len = len(pkts) rounds_per_timeslot = 1 + (pkt_cnt_per_timeslot // pkts_len) rounds_cnt = 0 + retry_failures = 0 while True: if terminated.is_set(): - logger.info("%d packets are sent", rounds_cnt * pkts_len) + logger.info("%d packets are sent (%d retry failures)", rounds_cnt * pkts_len, retry_failures) break logger.info("round %d, sending %d packets", rounds_cnt, rounds_cnt * pkts_len) for _ in range(rounds_per_timeslot): for pkt in pkts: - ptf_dataplane.send(device_num, port_num, pkt) + if not _send_with_retry(ptf_dataplane, device_num, port_num, pkt): + retry_failures += 1 while datetime.datetime.now() - last_round_time < datetime.timedelta(seconds=sending_timeslot): time.sleep(sending_timeslot / 10.0) diff --git a/tests/bgp/test_ipv6_nlri_over_ipv4.py b/tests/bgp/test_ipv6_nlri_over_ipv4.py index 9afcf4a920a..89660de127c 100644 --- a/tests/bgp/test_ipv6_nlri_over_ipv4.py +++ b/tests/bgp/test_ipv6_nlri_over_ipv4.py @@ -27,15 +27,16 @@ @pytest.fixture(scope='module') def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, request): neighbor_type = request.config.getoption("neighbor_type") - if neighbor_type not in ["sonic", "eos"]: + if neighbor_type not in ["sonic", "csonic", "eos"]: pytest.skip("Unsupported neighbor type: {}".format(neighbor_type)) is_sonic_neigh = True - if neighbor_type != "sonic": + if neighbor_type not in ("sonic", "csonic"): is_sonic_neigh = False duthost = duthosts[enum_frontend_dut_hostname] dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + confed_asn = duthost.get_bgp_confed_asn() lldp_table = duthost.shell("show lldp table")['stdout'].split("\n")[3].split() neigh_name = lldp_table[1] @@ -80,10 +81,16 @@ def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, request): neigh_asn[v['description']] = v['remote AS'] logger.debug(v['description']) + # FRR / vtysh needs the advertised dut_asn, not the confed ASN + dut_frr_asn = int(dut_asn) + if (neigh_ip_v4 is None or neigh_ip_v6 is None or peer_group_v4 is None or peer_group_v6 is None or neigh_asn is None): pytest.skip("Failed to get neighbor info") + neigh_bgp_config = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp'] + peer_in_bgp_confed = neigh_bgp_config.get('peer_in_bgp_confed', False) + dut_asn = int(confed_asn) if peer_in_bgp_confed else int(dut_asn) dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][dut_asn][0] dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][dut_asn][1].lower() @@ -143,6 +150,7 @@ def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, request): 'neighhost': nbrhosts[neigh_name]["host"], 'neigh_name': neigh_name, 'dut_asn': dut_asn, + 'dut_frr_asn': dut_frr_asn, 'neigh_asn': neigh_asn[neigh_name], 'namespace': namespace, 'dut_ip_v4': dut_ip_v4, @@ -203,7 +211,7 @@ def test_nlri(setup): # remove current neighbor adjacency cmd = 'vtysh {} -c "config" -c "router bgp {}" -c "no neighbor {} peer-group {}" \ -c "no neighbor {} peer-group {}"'\ - .format(setup['vtysh_ns'], setup['dut_asn'], setup['neigh_ip_v4'], setup['peer_group_v4'], + .format(setup['vtysh_ns'], setup['dut_frr_asn'], setup['neigh_ip_v4'], setup['peer_group_v4'], setup['neigh_ip_v6'], setup['peer_group_v6']) setup['duthost'].shell(cmd, module_ignore_errors=True) logger.debug("DUT BGP Config After Neighbor Removal: {}".format(setup['duthost'].shell('show run bgp')['stdout'])) @@ -286,13 +294,13 @@ def test_nlri(setup): -c "neighbor NLRI allowas-in" -c "neighbor NLRI send-community both" \ -c "neighbor NLRI soft-reconfiguration inbound" -c "exit-address-family" -c "address-family ipv6 unicast" \ -c "neighbor NLRI allowas-in" -c "neighbor NLRI send-community both" \ - -c "neighbor NLRI soft-reconfiguration inbound"'.format(setup['vtysh_ns'], setup['dut_asn']) + -c "neighbor NLRI soft-reconfiguration inbound"'.format(setup['vtysh_ns'], setup['dut_frr_asn']) setup['duthost'].shell(cmd, module_ignore_errors=True) cmd = 'vtysh {} -c "config" -c "router bgp {}" -c "neighbor {} peer-group NLRI" -c "neighbor {} remote-as {}"\ -c "address-family ipv4 unicast" -c "neighbor NLRI activate" -c "exit-address-family" \ -c "address-family ipv6 unicast" -c "neighbor NLRI activate"'\ - .format(setup['vtysh_ns'], setup['dut_asn'], setup['neigh_ip_v4'], setup['neigh_ip_v4'], + .format(setup['vtysh_ns'], setup['dut_frr_asn'], setup['neigh_ip_v4'], setup['neigh_ip_v4'], setup['neigh_asn']) setup['duthost'].shell(cmd, module_ignore_errors=True) logger.debug("DUT BGP Config After Peer Config: {}".format(setup['duthost'].shell('show run bgp')['stdout'])) diff --git a/tests/bgp/test_passive_peering.py b/tests/bgp/test_passive_peering.py index 30e532a07f4..7cc9be674ed 100644 --- a/tests/bgp/test_passive_peering.py +++ b/tests/bgp/test_passive_peering.py @@ -35,6 +35,7 @@ def setup(tbinfo, nbrhosts, duthosts, rand_one_dut_front_end_hostname, request): duthost = duthosts[rand_one_dut_front_end_hostname] dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + confed_asn = duthost.get_bgp_confed_asn() lldp_table = duthost.shell("show lldp table")['stdout'].split("\n")[3].split() neigh_name = lldp_table[1] @@ -69,8 +70,14 @@ def setup(tbinfo, nbrhosts, duthosts, rand_one_dut_front_end_hostname, request): assert v['state'] == 'established' neigh_asn[v['description']] = v['remote AS'] - dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][dut_asn][0] - dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][dut_asn][1] + neigh_bgp_config = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp'] + peer_in_bgp_confed = neigh_bgp_config.get('peer_in_bgp_confed', False) + if peer_in_bgp_confed: + asn = int(confed_asn) + else: + asn = int(dut_asn) + dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][asn][0] + dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh_name]['bgp']['peers'][asn][1] # EOS/cEOS converged: eos_config parents (nbrhosts flag or tbinfo convergence_data fallback) if is_sonic: diff --git a/tests/bgp/test_prefix_list.py b/tests/bgp/test_prefix_list.py index b2d0393e296..6378695afa8 100644 --- a/tests/bgp/test_prefix_list.py +++ b/tests/bgp/test_prefix_list.py @@ -150,7 +150,7 @@ def check_route_receive(prefix, expected_community, unexpected_community, neighb output_json = node["host"].get_route(prefix) logger.info("Neighbor {} route info: {}".format(node["host"].hostname, output_json)) hostname = node["host"].hostname - if neighbor_type == "sonic": + if neighbor_type in ("sonic", "csonic"): result = check_sonic_route_receive(present, output_json, expected_community, unexpected_community) elif neighbor_type == "eos": result = check_eos_route_receive(present, output_json, expected_community, unexpected_community, hostname, diff --git a/tests/bgp/test_prefix_list_suppress.py b/tests/bgp/test_prefix_list_suppress.py index 0d9aee19878..eaed084a702 100644 --- a/tests/bgp/test_prefix_list_suppress.py +++ b/tests/bgp/test_prefix_list_suppress.py @@ -15,6 +15,7 @@ import random import re import time +import uuid import pytest import yaml @@ -63,11 +64,6 @@ BGPCFGD_RUNNING_TIMEOUT = 60 BGPCFGD_RUNNING_INTERVAL = 5 -# Narrow journald lookback (seconds) used by TC-A5 after a fresh bgpcfgd -# restart. Wide enough to absorb the supervisorctl restart + bgpcfgd init -# on a slow real DUT (~30s p99), tight enough that stale lines from a -# previous image upgrade in the same 24h cannot leak into our assertions. -BGPCFGD_LOG_WINDOW_SECONDS = 180 # --------------------------------------------------------------------------- @@ -291,26 +287,63 @@ def start_bgp_container(duthost, service, container): duthost.shell("sudo docker start {}".format(container), module_ignore_errors=True) +def wait_for_frr_ready(duthost, container, asic_index, timeout=480): + """Wait until FRR vtysh responds on a specific ASIC. + + bgpcfgd RUNNING does NOT imply FRR daemons are ready — on T2 KVM + with slow I/O and large configs, FRR can take 6-7 minutes to open + its VTY socket. Uses 'timeout 10' to prevent vtysh from blocking + indefinitely when the socket is not yet available. + """ + ns = asic_ns_for_vtysh(duthost, asic_index) + cmd = 'timeout 10 vtysh {} -c "show version"'.format(ns).strip() + pytest_assert( + wait_until(timeout, 15, 0, + lambda c=cmd: duthost.shell(c, module_ignore_errors=True)["rc"] == 0), + "FRR vtysh not responsive within {}s on {} of {}".format( + timeout, container, duthost.hostname), + ) + + def restart_bgp_container(duthost): - """Restart the bgp container on every frontend asic and wait for bgpcfgd. - - ``sudo systemctl restart bgp`` occasionally exits non-zero on real DUTs - when systemd races with docker (the unit's start helper returns before - the container has fully come up). The bgp container itself still comes - up correctly a few seconds later, so we tolerate a non-zero rc here and - rely on :func:`wait_for_bgpcfgd` to confirm that bgpcfgd is actually - RUNNING. On some builds a failed restart can leave the container stopped, - so we do a bounded start fallback before failing.""" + """Restart BGP containers on all frontend ASICs in parallel, + then verify each ASIC's readiness sequentially. + + Restarts all BGP services first (parallel boot), then for each ASIC: + 1. Waits for bgpcfgd to be RUNNING (fast — Python daemon) + 2. Waits for FRR vtysh to respond (slower — needs VTY socket ready) + + The vtysh check uses 'timeout 10' to avoid indefinite blocking and + adapts to system speed: fast on physical hardware (<30s), slower on + T2 KVM (~7 min due to slow I/O and large config). + """ service_container_pairs = bgp_service_container_pairs(duthost) + + # 1. Restart all ASICs in parallel for service, _ in service_container_pairs: duthost.shell( "sudo systemctl restart {}".format(service), module_ignore_errors=True, ) - if not wait_until(60, BGPCFGD_RUNNING_INTERVAL, 0, bgpcfgd_running, duthost): - for service, container in service_container_pairs: + + # 2. Verify each ASIC sequentially: bgpcfgd up → vtysh responsive + sonichost = getattr(duthost, "sonichost", duthost) + for service, container in service_container_pairs: + suffix = container.removeprefix("bgp") + asic_index = int(suffix) if suffix.isdigit() else None + + # 2a. bgpcfgd running (fallback to explicit start if needed) + if not wait_until(60, BGPCFGD_RUNNING_INTERVAL, 0, + lambda c=container: sonichost.is_service_running("bgpcfgd", c)): start_bgp_container(duthost, service, container) - wait_for_bgpcfgd(duthost, timeout=180) + pytest_assert( + wait_until(180, BGPCFGD_RUNNING_INTERVAL, 0, + lambda c=container: sonichost.is_service_running("bgpcfgd", c)), + "bgpcfgd not running in {} on {}".format(container, duthost.hostname), + ) + + # 2b. FRR vtysh responsive + wait_for_frr_ready(duthost, container, asic_index) def apply_constants_to_bgpcfgd(duthost): @@ -357,40 +390,34 @@ def apply_constants_to_bgpcfgd(duthost): ) -def restart_bgpcfgd_only(duthost): - """Bounce just the ``bgpcfgd`` supervisord program in every frontend - ASIC's bgp container and wait for it to come back RUNNING. +def place_syslog_marker(duthost): + """Inject a unique marker into /var/log/syslog and return the marker string. - Cheaper and more deterministic than a full ``systemctl restart bgp``: - it keeps the container, zebra, bgpd and staticd alive and only - re-runs bgpcfgd's ``main()``/``do_work()``. The one-shot - ``log_notice`` startup lines (``AsPath Manager is enabled for - ``, etc.) are re-emitted so the caller can use a - narrow ``--since`` window for syslog assertions. + Used to window syslog assertions: after placing the marker, only lines + after it are considered by collect_syslog_since_marker(). + Waits until the marker is confirmed present in the syslog file to avoid + races with asynchronous rsyslog writes. """ - for container in bgp_container_names(duthost): - duthost.shell( - "sudo docker exec {} supervisorctl restart bgpcfgd".format(container), - module_ignore_errors=True, - ) - wait_for_bgpcfgd(duthost, timeout=180) + marker = "SONIC_TEST_MARKER_{}".format(uuid.uuid4().hex[:12]) + duthost.shell("logger -t SONIC_TEST '{}'".format(marker)) + duthost.shell( + "timeout 5 bash -c \"while ! grep -q '{}' /var/log/syslog; do sleep 0.1; done\"".format(marker) + ) + return marker -def collect_recent_syslog(duthost, pattern, since_seconds=120): - """Return matching lines from the last *since_seconds* of /var/log/syslog. +def collect_syslog_since_marker(duthost, marker, pattern): + """Return lines from /var/log/syslog matching *pattern* after *marker*. - The grep filters out ansible's own audit log lines - (``ansible-ansible.legacy.command Invoked with _raw_params=...``), which - echo the journalctl command itself, including the regex pattern, - back into the same log we're searching. Without this filter every call - would appear to match its own argument and produce a false positive.""" - # The bgpcfgd warnings flow into the host's syslog; multi-asic containers - # also forward their syslog to the host. We just grep the host log. + Works on both single-ASIC and multi-ASIC platforms because all container + daemons forward their logs to the host's /var/log/syslog via rsyslog. + Filters out ansible audit lines that echo the grep pattern back. + """ cmd = ( - "sudo journalctl --since='{}sec ago' --no-pager 2>/dev/null " - "| grep -v 'ansible-ansible.legacy.command Invoked with _raw_params' " - "| grep -E {!r} || true" - ).format(since_seconds, pattern) + "sudo sed -n '/{marker}/,$p' /var/log/syslog " + "| grep -v 'Invoked with _raw_params' " + "| grep -E {pattern!r} || true" + ).format(marker=marker, pattern=pattern) return duthost.shell(cmd)["stdout"] @@ -654,68 +681,69 @@ def test_prefix_list_mgr_running_on_every_device(self, rand_one_frontend_duthost """TC-A5: PrefixListMgr is now started unconditionally; old log line must be gone; bgpcfgd must be healthy. - We deliberately bounce ``bgpcfgd`` at the start of this test and - use a narrow post-restart window (``BGPCFGD_LOG_WINDOW_SECONDS``) - for every syslog assertion. A 24h ``journalctl`` window would - otherwise surface stale lines from a *previous* image (e.g. the - legacy ``Prefix List Manager and AsPath Manager are enabled for - UpperSpineRouter/UpstreamLC`` notice or a bgpcfgd traceback that - happened before today's upgrade), which would make this test - fail spuriously on regression cycles that re-image the DUT. - Bouncing bgpcfgd also forces the one-shot ``log_notice`` startup - lines (in particular ``AsPath Manager is enabled for - `` on spines) to be re-emitted into the narrow - window, so the presence/absence checks below are deterministic. + We restart ``bgpcfgd`` one container at a time, placing a unique + syslog marker before each restart. This scopes every assertion to + exactly the log output from that container's fresh bgpcfgd startup, + eliminating false positives from stale lines in a previous image. + Uses /var/log/syslog (not journalctl) so it works on multi-ASIC + chassis where container daemons log via rsyslog. """ duthost = rand_one_frontend_duthost + spine = is_upstream_spine(duthost) + + for container in bgp_container_names(duthost): + # Place marker, restart bgpcfgd, wait for it to come back + marker = place_syslog_marker(duthost) + duthost.shell( + "sudo docker exec {} supervisorctl restart bgpcfgd".format( + container), + module_ignore_errors=True, + ) + sonichost = getattr(duthost, "sonichost", duthost) + pytest_assert( + wait_until(180, 5, 0, + lambda c=container: sonichost.is_service_running( + "bgpcfgd", c)), + "bgpcfgd not RUNNING in {} within 180s".format(container), + ) - restart_bgpcfgd_only(duthost) - # ``wait_for_bgpcfgd`` inside ``restart_bgpcfgd_only`` already - # asserts bgpcfgd is RUNNING on every frontend ASIC. - - # The pre-PR notice was tied to UpperSpineRouter/UpstreamLC and - # should no longer appear at all -- even after a fresh bgpcfgd - # startup. - legacy = collect_recent_syslog( - duthost, - "Prefix List Manager and AsPath Manager are enabled for " - "UpperSpineRouter/UpstreamLC", - since_seconds=BGPCFGD_LOG_WINDOW_SECONDS, - ) - pytest_assert( - not legacy.strip(), - "Legacy 'Prefix List Manager and AsPath Manager are enabled' log " - "line is still produced after bgpcfgd restart on {}:\n{}".format( - duthost.hostname, legacy), - ) - - # On spine devices the new "AsPath Manager is enabled for " line - # must appear. On non-spine devices it must not. - new_line = collect_recent_syslog( - duthost, - "AsPath Manager is enabled for", - since_seconds=BGPCFGD_LOG_WINDOW_SECONDS, - ) - if is_upstream_spine(duthost): + # The pre-PR notice was tied to UpperSpineRouter/UpstreamLC and + # should no longer appear at all after a fresh bgpcfgd startup. + legacy = collect_syslog_since_marker( + duthost, marker, + "Prefix List Manager and AsPath Manager are enabled for " + "UpperSpineRouter/UpstreamLC", + ) pytest_assert( - new_line.strip(), - "Expected 'AsPath Manager is enabled for ' log " - "line on spine device {} after bgpcfgd restart".format( - duthost.hostname), + not legacy.strip(), + "Legacy 'Prefix List Manager and AsPath Manager are enabled'" + " log line still produced after bgpcfgd restart in {} on" + " {}:\n{}".format(container, duthost.hostname, legacy), ) - # bgpcfgd must not have logged a traceback since the restart we - # just performed. We use the same narrow window so a stale - # traceback from a previous run cannot fail this assertion. - tracebacks = collect_recent_syslog( - duthost, "bgpcfgd.*Traceback", - since_seconds=BGPCFGD_LOG_WINDOW_SECONDS, - ) - pytest_assert( - not tracebacks.strip(), - "bgpcfgd recorded a traceback after restart on {}:\n{}".format( - duthost.hostname, tracebacks), - ) + # On spine devices the new "AsPath Manager is enabled for " + # line must appear. On non-spine devices it must not. + if spine: + new_line = collect_syslog_since_marker( + duthost, marker, + "AsPath Manager is enabled for", + ) + pytest_assert( + new_line.strip(), + "Expected 'AsPath Manager is enabled for '" + " log line on spine device {} after bgpcfgd restart in" + " {}".format(duthost.hostname, container), + ) + + # bgpcfgd must not have logged a traceback since the restart. + tracebacks = collect_syslog_since_marker( + duthost, marker, "bgpcfgd.*Traceback", + ) + pytest_assert( + not tracebacks.strip(), + "bgpcfgd recorded a traceback after restart in {} on" + " {}:\n{}".format(container, duthost.hostname, tracebacks), + ) # --------------------------------------------------------------------------- @@ -826,6 +854,7 @@ def test_suppress_prefix_on_non_spine_device( # --- Direct CONFIG_DB path --- try: + marker = place_syslog_marker(duthost) write_config_db_key_directly(duthost, SUPPRESS_TYPE, prefix) pytest_assert( wait_until( @@ -835,11 +864,10 @@ def test_suppress_prefix_on_non_spine_device( ), "SUPPRESS_PREFIX not picked up by PrefixListMgr from direct DB write", ) - unwanted_warn = collect_recent_syslog( - duthost, + unwanted_warn = collect_syslog_since_marker( + duthost, marker, "PrefixListMgr:: Device type .* not supported for {}".format( SUPPRESS_TYPE), - since_seconds=60, ) pytest_assert( not unwanted_warn.strip(), diff --git a/tests/bgp/test_traffic_shift_lc.py b/tests/bgp/test_traffic_shift_lc.py index bf2af2fba49..5b2662b1928 100644 --- a/tests/bgp/test_traffic_shift_lc.py +++ b/tests/bgp/test_traffic_shift_lc.py @@ -16,7 +16,8 @@ from tests.bgp.traffic_checker import get_traffic_shift_state, check_tsa_persistence_support, \ verify_traffic_shift_per_asic from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE, TS_NO_NEIGHBORS -from tests.conftest import get_hosts_per_hwsku +from tests.conftest import get_hosts_per_hwsku, backup_golden_config, restore_golden_config, \ + update_golden_config_tsa_enabled pytestmark = [ pytest.mark.topology('t2') @@ -360,9 +361,15 @@ def test_load_minigraph_with_traffic_shift_away(request, duthosts, nbrhosts, tra # Initially make sure both supervisor and line cards are in BGP operational normal state initial_tsa_check_before_and_after_test(duthosts) + # Backup and update golden_config_db.json to enable TSA + backup_path = "/tmp/golden_config_db_backup.json" + for duthost in frontend_nodes_per_hwsku: + backup_golden_config(duthost, backup_path) + update_golden_config_tsa_enabled(duthost, tsa_enabled=True) + for duthost in frontend_nodes_per_hwsku: # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(wait_until(30, 5, 0, lambda: TS_NORMAL == get_traffic_shift_state(duthost, 'TSC no-stats')), + pytest_assert(wait_until(60, 5, 0, lambda: TS_NORMAL == get_traffic_shift_state(duthost, 'TSC no-stats')), "DUT is not in normal state") if not check_tsa_persistence_support(duthost): pytest.skip("TSA persistence not supported in the image") @@ -381,7 +388,7 @@ def test_load_minigraph_with_traffic_shift_away(request, duthosts, nbrhosts, tra for duthost in frontend_nodes_per_hwsku: # Verify DUT is in maintenance state. - pytest_assert(wait_until(30, 5, 0, + pytest_assert(wait_until(60, 5, 0, lambda: TS_MAINTENANCE == get_traffic_shift_state(duthost, 'TSC no-stats')), "DUT is not in maintenance state") assert_only_loopback_routes_announced_to_neighs( @@ -402,10 +409,14 @@ def test_load_minigraph_with_traffic_shift_away(request, duthosts, nbrhosts, tra # Verify DUT is in normal state. for duthost in frontend_nodes_per_hwsku: - pytest_assert(wait_until(30, 5, 0, + pytest_assert(wait_until(60, 5, 0, lambda: TS_NORMAL == get_traffic_shift_state(duthost, 'TSC no-stats')), "DUT is not in normal state") + # Restore the original golden_config_db.json + for duthost in frontend_nodes_per_hwsku: + restore_golden_config(duthost, backup_path) + # Wait until all routes are announced to neighbors verify_route_on_neighbors(frontend_nodes_per_hwsku, dut_nbrhosts, orig_v4_routes, orig_v6_routes) # Bring back the supervisor and line cards to the BGP operational normal state diff --git a/tests/bmp/test_frr_bmp_sanity.py b/tests/bmp/test_frr_bmp_sanity.py index adee9428b57..9f4a49f907c 100644 --- a/tests/bmp/test_frr_bmp_sanity.py +++ b/tests/bmp/test_frr_bmp_sanity.py @@ -1,4 +1,5 @@ import pytest +from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.monit import check_monit_expected_container_logging from tests.common.utilities import wait_until from bmp.helper import enable_bmp_feature, disable_bmp_feature @@ -13,6 +14,7 @@ def test_frr_bmp_monit_log(duthosts, enum_frontend_dut_hostname, enum_asic_index duthost = duthosts[enum_frontend_dut_hostname] disable_bmp_feature(duthost) - wait_until(180, 60, 0, check_monit_expected_container_logging, duthost) + pytest_assert(wait_until(180, 60, 0, check_monit_expected_container_logging, duthost), + "Monit logged unexpected container-not-running messages") enable_bmp_feature(duthost) diff --git a/tests/common/connections/base_console_conn.py b/tests/common/connections/base_console_conn.py index 84043da7a59..b1197a52b11 100644 --- a/tests/common/connections/base_console_conn.py +++ b/tests/common/connections/base_console_conn.py @@ -3,8 +3,10 @@ """ import logging +import paramiko from netmiko.cisco_base_connection import CiscoBaseConnection + try: from netmiko.ssh_exception import NetMikoAuthenticationException except ImportError: @@ -17,6 +19,8 @@ import tty import select +logger = logging.getLogger(__name__) + # All supported console types # Console login via telnet (mad console) CONSOLE_TELNET = "console_telnet" @@ -48,6 +52,46 @@ def __init__(self, **kwargs): if key in kwargs: del kwargs[key] + # Allow legacy KEX and host key algorithms for older console servers + # (e.g. Cisco SSH-2.0-Cisco-1.25) that only support legacy crypto. + # paramiko 5.x removed these from _preferred_kex, _kex_info, + # _preferred_keys, _key_info, and RSAKey.HASHES. + try: + from paramiko.kex_group14 import KexGroup14SHA256 + from paramiko.kex_gex import KexGexSHA256 + from paramiko.rsakey import RSAKey + from hashlib import sha1 as _sha1 + from cryptography.hazmat.primitives.hashes import SHA1 + + # Reconstruct legacy KEX classes from existing SHA256 variants + class _KexGroup14SHA1(KexGroup14SHA256): + name = "diffie-hellman-group14-sha1" + hash_algo = _sha1 + + class _KexGexSHA1(KexGexSHA256): + name = "diffie-hellman-group-exchange-sha1" + hash_algo = _sha1 + + _legacy_kex = { + "diffie-hellman-group14-sha1": _KexGroup14SHA1, + "diffie-hellman-group-exchange-sha1": _KexGexSHA1, + } + for kex_name, kex_cls in _legacy_kex.items(): + if kex_name not in paramiko.Transport._preferred_kex: + paramiko.Transport._preferred_kex += (kex_name,) + if kex_name not in paramiko.Transport._kex_info: + paramiko.Transport._kex_info[kex_name] = kex_cls + + # Re-enable ssh-rsa host key support + if "ssh-rsa" not in paramiko.Transport._preferred_keys: + paramiko.Transport._preferred_keys += ("ssh-rsa",) + if "ssh-rsa" not in paramiko.Transport._key_info: + paramiko.Transport._key_info["ssh-rsa"] = RSAKey + if "ssh-rsa" not in RSAKey.HASHES: + RSAKey.HASHES["ssh-rsa"] = SHA1 + except Exception as e: + logger.debug("Could not re-enable legacy SSH algorithms for console: %s", e) + for i in range(0, len(all_passwords)): kwargs['password'] = all_passwords[i] try: diff --git a/tests/common/dash_utils.py b/tests/common/dash_utils.py index e1dd07f5856..07444d34fba 100644 --- a/tests/common/dash_utils.py +++ b/tests/common/dash_utils.py @@ -1,4 +1,6 @@ import logging +import re +from enum import IntEnum from os import path from time import sleep @@ -13,6 +15,200 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- # +# DASH config ordering utility +# +# DASH objects pushed to the DPU via gNMI (``gnmi_utils.apply_messages``) have +# inter-table dependencies (e.g. ``DASH_ENI_TABLE`` references a +# ``DASH_VNET_TABLE`` and ``DASH_METER_POLICY_TABLE``; +# ``DASH_ENI_ROUTE_TABLE`` binds an ENI to a ``DASH_ROUTE_GROUP_TABLE``). +# Historically each test fixture has manually re-implemented the same +# table-bucketing-and-apply order. ``apply_dash_configs`` below centralises +# that policy: callers pass any number of config dicts keyed by +# ``DASH__TABLE:`` and the helper groups + applies them in +# the dependency order defined by ``DASH_TABLE_PHASE``. +# +# To add a new DASH table type, add an entry to ``DASH_TABLE_PHASE`` choosing +# the ``DashPhase`` that satisfies its dependencies. To make an entry land in +# a non-default phase from a specific call site, split it into its own dict +# and rely on the registry; per-call overrides intentionally aren't supported. +# --------------------------------------------------------------------------- # + + +class DashPhase(IntEnum): + """Phases for applying DASH configurations in dependency order. + + Lower-numbered phases are applied first on setup and deleted last on + teardown. Two table names may share a phase if they have no ordering + dependency relative to each other (or if their inter-dependency is + handled within a single gNMI batch by orchagent/SAI). + + Phase contents (canonical ordering): + - ``GROUP_1`` — APPLIANCE + - ``GROUP_2`` — ROUTING_TYPE, METER_POLICY, OUTBOUND_PORT_MAP, VNET + - ``GROUP_3`` — METER_RULE + - ``GROUP_4`` — TUNNEL, OUTBOUND_PORT_MAP_RANGE, ENI, ROUTE_GROUP + - ``GROUP_5`` — ROUTE_RULE, ROUTE, VNET_MAPPING + - ``GROUP_6`` — ENI_ROUTE + """ + GROUP_1 = 1 + GROUP_2 = 2 + GROUP_3 = 3 + GROUP_4 = 4 + GROUP_5 = 5 + GROUP_6 = 6 + + +DASH_TABLE_PHASE = { + "APPLIANCE": DashPhase.GROUP_1, + "ROUTING_TYPE": DashPhase.GROUP_2, + "METER_POLICY": DashPhase.GROUP_2, + "OUTBOUND_PORT_MAP": DashPhase.GROUP_2, + "VNET": DashPhase.GROUP_2, + "METER_RULE": DashPhase.GROUP_3, + "TUNNEL": DashPhase.GROUP_4, + "OUTBOUND_PORT_MAP_RANGE": DashPhase.GROUP_4, + "ENI": DashPhase.GROUP_4, + "ROUTE_GROUP": DashPhase.GROUP_4, + "ROUTE_RULE": DashPhase.GROUP_5, + "ROUTE": DashPhase.GROUP_5, + "VNET_MAPPING": DashPhase.GROUP_5, + "ENI_ROUTE": DashPhase.GROUP_6, +} + +# Phase used for any ``DASH__TABLE`` key not present in +# ``DASH_TABLE_PHASE``. Defaulting to the latest phase makes new tables +# effectively "apply last / delete first", which is the safer fallback when +# their real dependencies aren't yet known. +DEFAULT_DASH_PHASE = DashPhase.GROUP_6 + +# Captures DASH__TABLE at the start of a redis-style key. +# Greedy ``\w+`` correctly handles multi-word table names such as +# ``DASH_OUTBOUND_PORT_MAP_RANGE_TABLE``. +_DASH_KEY_RE = re.compile(r"^DASH_(\w+)_TABLE(?::|$)") + + +def dash_table_name(key): + """Extract the DASH table name from a redis-style DASH key. + + Args: + key: A string like ``"DASH_VNET_MAPPING_TABLE:Vnet1:10.0.0.1"`` or + just ``"DASH_APPLIANCE_TABLE"``. + + Returns: + The table name without the ``DASH_`` prefix or ``_TABLE`` suffix + (e.g. ``"VNET_MAPPING"``, ``"APPLIANCE"``). + + Raises: + ValueError: if ``key`` does not match the DASH table key shape. + """ + m = _DASH_KEY_RE.match(key) + if not m: + raise ValueError("Not a DASH table key: {!r}".format(key)) + return m.group(1) + + +def bucket_dash_configs(*config_dicts): + """Merge config dicts and group entries by their :class:`DashPhase`. + + Entries from later dicts that share a key with earlier dicts overwrite + the earlier value (the same semantics as ``{**a, **b}``), but a warning + is logged when the two values differ so silent config drift is visible. + + Args: + *config_dicts: Any number of dicts keyed by + ``"DASH__TABLE:"``. + + Returns: + A list of ``(phase, merged_dict)`` tuples sorted by ascending phase. + Empty if no input entries were supplied. + + Raises: + ValueError: if any key is not a DASH table key. + """ + by_phase = {} + seen = {} + for d in config_dicts: + for k, v in d.items(): + if k in seen and seen[k] != v: + logger.warning( + "Duplicate DASH key %r with conflicting values across " + "input dicts; later value wins", + k, + ) + seen[k] = v + tbl = dash_table_name(k) + phase = DASH_TABLE_PHASE.get(tbl) + if phase is None: + logger.warning( + "Unknown DASH table %r in key %r; defaulting to phase %s. " + "Add an entry to DASH_TABLE_PHASE to silence this " + "warning.", + tbl, k, DEFAULT_DASH_PHASE.name, + ) + phase = DEFAULT_DASH_PHASE + by_phase.setdefault(phase, {})[k] = v + return sorted(by_phase.items(), key=lambda item: item[0]) + + +def apply_dash_configs( + localhost, duthost, ptfhost, dpu_index, *config_dicts, + set_db=True, wait_after_apply=5, max_updates_in_single_cmd=1024, + apply_fn=None, +): + """Apply DASH configs to the DPU in dependency order based on table name. + + Buckets entries from all ``config_dicts`` by :class:`DashPhase` (see + ``DASH_TABLE_PHASE``) and calls the underlying gNMI apply function once + per non-empty phase, in ascending phase order on setup or descending + order when ``set_db=False`` (delete) so dependents are removed first. + + Args: + localhost, duthost, ptfhost, dpu_index: passed through to ``apply_fn``. + *config_dicts: Any number of dicts keyed by + ``"DASH_
_TABLE:"``. Empty / ``None`` dicts are + tolerated and skipped (so callers can use conditional inclusion + patterns like ``*(extra if condition else [])``). + set_db: ``True`` (default) to write configs; ``False`` to delete. + wait_after_apply: seconds to wait after each phase's apply; forwarded + to ``apply_fn`` per phase. + max_updates_in_single_cmd: forwarded to ``apply_fn``. + apply_fn: optional callable matching the signature of + ``gnmi_utils.apply_messages``. Defaults to a lazy import so this + module does not require ``gnmi_utils`` to be on ``sys.path`` at + import time. Pass an injectable fake for unit tests. + """ + if apply_fn is None: + # Lazy import: ``tests/common/dash_utils.py`` is imported by both + # DASH and HA tests, each of which provides its own ``gnmi_utils`` + # module on ``sys.path`` with a compatible ``apply_messages``. + from gnmi_utils import apply_messages as _default_apply_fn + apply_fn = _default_apply_fn + + non_empty = [d for d in config_dicts if d] + if not non_empty: + logger.info("apply_dash_configs called with no entries; nothing to do") + return + + buckets = bucket_dash_configs(*non_empty) + if not set_db: + buckets = list(reversed(buckets)) + + op_label = "SET" if set_db else "DEL" + for phase, messages in buckets: + tables = sorted({dash_table_name(k) for k in messages}) + logger.info( + "[%s] DASH phase %s (priority %d): %d entries across tables %s", + op_label, phase.name, int(phase), len(messages), tables, + ) + apply_fn( + localhost, duthost, ptfhost, messages, dpu_index, + set_db=set_db, + wait_after_apply=wait_after_apply, + max_updates_in_single_cmd=max_updates_in_single_cmd, + ) + + def safe_open_template(template_path): """ Safely loads Jinja2 template from given path @@ -116,16 +312,16 @@ def verify_tunnel_packets(ptfadapter, ports, exp_dpu_to_vm_pkt, tunnel_endpoint_ if pkt_repr["IP"].dst in tunnel_endpoint_counts: tunnel_endpoint_counts[pkt_repr["IP"].dst] += 1 logging.debug( - f"Packet sent to tunnel endpoint {pkt_repr['IP'].dst} matches:\ - \n{result.format()} \nExpected:\n{exp_dpu_to_vm_pkt}" + f"Packet sent to tunnel endpoint {pkt_repr['IP'].dst} matches: \ + \n{result.format()} \nExpected: \n{exp_dpu_to_vm_pkt}" ) return else: pytest.fail( f"Received packet has unexpected dst IP {pkt_repr['IP'].dst}, \ expected one of {tunnel_endpoint_counts.keys()} \ - \n{result.format()} \nExpected:\n{exp_dpu_to_vm_pkt}" + \n{result.format()} \nExpected: \n{exp_dpu_to_vm_pkt}" ) else: pytest.fail(f"Got expected packet on unexpected port {result.port}: {pkt_repr}") - pytest.fail(f"DP poll failed:\n{result.format()}") + pytest.fail(f"DP poll failed: \n{result.format()}") diff --git a/tests/common/devices/arista.py b/tests/common/devices/arista.py index b8c38025d1d..6ca6f508a7e 100644 --- a/tests/common/devices/arista.py +++ b/tests/common/devices/arista.py @@ -101,7 +101,7 @@ def get_lldp_neighbor(self, local_iface=None): else: command = "show lldp neighbors | json" output = self.commands(commands=[command]) - return output["stdout_lines"][0] if output["failed"] is False else False + return output["stdout_lines"][0] if output.get("failed", False) is False else False except Exception as e: logger.error("command {} failed. exception: {}".format(command, repr(e))) return False @@ -228,7 +228,7 @@ def get_lldp_neighbors(self): logger.info("Gathering LLDP details") command = "show lldp neighbors | json" json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): for row in json_output["stdout_lines"][0]["lldpNeighbors"]: lldp_details.update( { @@ -262,7 +262,7 @@ def get_all_lldp_neighbor_details_for_port(self, physical_port): command = "show lldp neighbors {} detail | json".format(physical_port) try: json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0] return "Failed to get lldp neighbor details due to {}".format(json_output) except Exception as e: @@ -276,7 +276,7 @@ def get_chassis_id_from_cli(self): try: command = "show lldp local-info | json" json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0]["chassisId"] return "Failed to get chassis id due to {}".format(json_output) except Exception as e: @@ -289,7 +289,7 @@ def get_mgmt_ip_from_cli(self): try: command = "show lldp local-info | json" json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0]["managementAddresses"][0]["address"] return "Failed to get mgmt IP due to {}".format(json_output) except Exception as e: @@ -302,7 +302,7 @@ def get_platform_from_cli(self): try: command = "show version | json" json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0]["modelName"] return "Failed to get platform info due to {}".format(json_output) except Exception as e: @@ -315,7 +315,7 @@ def get_version_from_cli(self): try: command = "show version | json" json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0]["version"] return "Failed to get platform info due to {}".format(json_output) except Exception as e: @@ -440,7 +440,7 @@ def get_all_interfaces_in_pc(self, pc_name): pc_on = pc_name.replace("Port-Channel", "") command = "show lacp {} aggregates | json".format(pc_on) json_output = self.commands(commands=[command]) - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout_lines"][0]["portChannels"][pc_name]["bundledPorts"] return "Failed to get interfaces in port chancel due to {}".format(json_output) except Exception as e: @@ -460,7 +460,7 @@ def check_interface_status(self, interface): try: success_criteria = "line protocol is up" intf_status_output = self.commands(commands=[command]) - if not intf_status_output["failed"]: + if not intf_status_output.get("failed", False): logger.info("Interface status check: {} sent to {}".format(command, self.hostname)) is_up = success_criteria in intf_status_output["stdout"][0].lower() return is_up, intf_status_output["stdout_lines"][0] @@ -475,7 +475,7 @@ def get_isis_adjacency(self): logger.info("Gathering ISIS adjacency details") command = "show isis neighbors| json" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): isis_instances = output["stdout"][0]["vrfs"]["default"]["isisInstances"].keys() for instance in isis_instances: for key, line in output["stdout"][0]["vrfs"]["default"]["isisInstances"][instance][ @@ -515,7 +515,7 @@ def get_isis_database(self, queue=None): command = "show isis database" output = self.commands(commands=[command]) lsp_entries = {} - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "00-0" in line: outline = line.strip().split() @@ -540,7 +540,7 @@ def get_bgp_status(self): try: output = self.commands(commands=[command]) bgp_status = {} - if not output["failed"]: + if not output.get("failed", False): bgp_peer_info = output["stdout"][0]["vrfs"]["default"]["peers"] for peer_ip, peer_info in bgp_peer_info.items(): bgp_status[peer_ip] = peer_info["peerState"] @@ -569,7 +569,7 @@ def get_bgp_session_status(self, peer_ip): """ try: bgp_peer_details = self.get_bgp_session_details(peer_ip) - if not bgp_peer_details["failed"]: + if not bgp_peer_details.get("failed", False): bgp_session_status = bgp_peer_details["stdout"][0]["vrfs"]["default"]["peerList"][0]["state"] else: @@ -588,7 +588,7 @@ def is_prefix_advertised_to_peer(self, prefix, peer_ip): try: json_output = self.commands(commands=[command]) prefix_adv_status = False - if not json_output["failed"]: + if not json_output.get("failed", False): bgp_route_entries = json_output["stdout"][0]["vrfs"]["default"]["bgpRouteEntries"] if len(bgp_route_entries) > 0 and prefix in bgp_route_entries: prefix_adv_status = True @@ -611,7 +611,7 @@ def check_remote_ldp_sessions(self): command = "show mpls ldp neighbor summary | json" json_output = self.commands(commands=[command]) ldp_op_list = [] - if not json_output["failed"]: + if not json_output.get("failed", False): for neighbor in json_output["stdout"][0]["vrfs"]["default"]["neighbors"]: if "state" in neighbor and neighbor["state"] == "stateOperational": ldp_op_list.append(neighbor["tcpPeerIp"]["ip"]) @@ -637,7 +637,7 @@ def get_core_interfaces(self, ldp_op_list): for ldpneighbor in ldp_op_list: command = "show ip route {}".format(ldpneighbor) output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "Port-Channel" in line: ldp_int_list.append(line.split(",")[1].strip()) @@ -751,7 +751,7 @@ def set_rekey_period(self, profile_name, rekey_period_value): commands = ["mka session rekey-period {}".format(rekey_period_value)] parents = ["mac security", "profile {}".format(profile_name)] output = self.config(lines=commands, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to set rekey period due to {}".format(output) except Exception as e: @@ -768,7 +768,7 @@ def get_macsec_profile(self, interface): """example of output: mac security profile macsec-profile-juniper-256-64CKN-64CAK-fallback """ - if not output["failed"]: + if not output.get("failed", False): return True, output["stdout_lines"][0][-1].split()[-1] return False, "Failed to get macsec profile due to {}".format(output) except Exception as e: @@ -788,7 +788,7 @@ def get_macsec_status_logs(self, interface, last_count="30", log_type="ESTABLISH return True, "Device {} not support {} log".format(self.hostname, log_type) command = "show logging all | grep MKA | grep {} | grep {}".format(interface, log_type) output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): return len(output["stdout_lines"][0]) > 0, output["stdout"][0] return False, "Failed to get macsec status logs due to {}".format(output) except Exception as e: @@ -848,7 +848,7 @@ def get_macsec_config(self, interface): """example of output: mac security profile macsec-profile-juniper-256-64CKN-64CAK-fallback """ - if not output["failed"]: + if not output.get("failed", False): # Returning only MACSEC config. for config in output["stdout_lines"][0]: if "security" in config: @@ -873,7 +873,7 @@ def apply_macsec_interface_config(self, commands): list_command.append(line[1]) if len(list_command) > 0: output = self.config(lines=commands, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to apply macsec interface config due to {}".format(output) return False, "Failed to apply macsec interface config due to no commmand available." @@ -889,7 +889,7 @@ def delete_macsec_interface_config(self, interface): parents = ["interface {}".format(interface)] commands = ["no mac security profile"] output = self.config(lines=commands, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to delete macsec interface config due to {}".format(output) except Exception as e: @@ -930,7 +930,7 @@ def get_macsec_interface_statistics(self, interface): } } """ - if not json_output["failed"]: + if not json_output.get("failed", False): counter = json_output["stdout"][0]["interfaces"][interface] validated_bytes = counter["countersDetail"]["inPktsOK"] decrypted_bytes = counter["inPktsDecrypted"] @@ -975,7 +975,7 @@ def get_loopback_ipv4_addr(self): try: command = "show running-config interfaces loopback 99" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "ip address" in line: lb_ipv4_addr = line.split()[-1].strip("/32") @@ -995,7 +995,7 @@ def remove_int_from_portchan(self, interface, pcnum): command = ["no channel-group"] parents = ["interface {}".format(interface)] output = self.config(lines=command, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, "remove interface {} from ether-bundle {}".format(interface, pcnum) else: return False, "Failed to remove interface {} from ether-bundle {}".format(interface, pcnum) @@ -1015,7 +1015,7 @@ def put_int_in_portchan(self, interface, pcnum): command = ["channel-group {} mode active".format(pcnum)] parents = ["interface {}".format(interface)] output = self.config(lines=command, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, "Added interface {} from channel-group {}".format(interface, pcnum) else: return False, "Failed to add interface {} to channel-group {}".format(interface, pcnum) @@ -1035,7 +1035,7 @@ def run_configure_command_test(self): parents = [] command = ["alias testversion show version"] output = self.config(lines=command, parents=parents) - if not output["failed"]: + if not output.get("failed", False): rollback_command = "no alias testversion" self.config(lines=[rollback_command], parents=parents) return True, output @@ -1099,7 +1099,7 @@ def check_for_aggregate_route_generation(self, agg_prefix): command = "show ip route aggregate | include {}".format(agg_prefix) agg_route_gen_status = False output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if agg_prefix in line: agg_route_gen_status = True @@ -1113,7 +1113,7 @@ def get_ipfix_export_data_count(self): packets_exported = 0 command = "show flow tracking sampled counters | include messages" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "messages" in line: counter = line.split()[1].lstrip("(").rstrip(")") @@ -1137,7 +1137,7 @@ def apply_sample_filter_to_interface(self, filter_name, interface): parents = ["interface {}".format(interface)] commands = ["flow tracker sampled {}".format(filter_name)] output = self.config(lines=commands, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1147,7 +1147,7 @@ def reboot_chassis(self): try: command = "reload all now" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: diff --git a/tests/common/devices/base.py b/tests/common/devices/base.py index ce1ef86494c..494e63c1193 100644 --- a/tests/common/devices/base.py +++ b/tests/common/devices/base.py @@ -247,6 +247,14 @@ def run_module(module_args, complex_args): if (hostname_res.is_failed or 'exception' in hostname_res) and not module_ignore_errors: raise RunAnsibleModuleFail("run module {} failed".format(module_name), hostname_res) + # Ensure 'failed' key is always present for backward compatibility with code that + # accesses rc['failed'] directly. The _IGNORE hack above is no longer effective in + # ansible-core >= 2.21 where the post-processing behavior changed so that 'failed' + # is absent from successful command results. Normalizing here is a single robust fix + # that covers all call sites without requiring per-file changes. + if 'failed' not in hostname_res: + hostname_res['failed'] = hostname_res.is_failed + return hostname_res diff --git a/tests/common/devices/cisco.py b/tests/common/devices/cisco.py index 393dab91045..cc2cfd07ab6 100644 --- a/tests/common/devices/cisco.py +++ b/tests/common/devices/cisco.py @@ -187,7 +187,9 @@ def get_lldp_neighbor(self, local_iface=None, remote_device=None): commands=[command], module_ignore_errors=True) logger.debug('cisco lldp output: %s' % (output)) - return output['stdout_lines'][0]['Response']['Get']['Operational'] if output['failed'] is False else False + if output.get('failed', False): + return False + return output['stdout_lines'][0]['Response']['Get']['Operational'] except Exception as e: logger.error('command {} failed. exception: {}'.format(command, repr(e))) return False @@ -239,7 +241,9 @@ def ping_dest(self, dest): command = 'ping {} count 5'.format(dest) output = self.commands(commands=[command]) logger.debug('ping result: %s' % (output)) - return re.search('!!!!!', output['stdout'][0]) is not None if output['failed'] is False else False + if output.get('failed', False): + return False + return re.search('!!!!!', output['stdout'][0]) is not None except Exception as e: logger.error('command {} failed. exception: {}'.format(command, repr(e))) return False @@ -387,7 +391,7 @@ def get_lldp_neighbors(self): header_line = "Device ID Local Intf Hold-time Capability Port ID" end_line = "Total entries displayed:" content_idx = 0 - if not output['failed']: + if not output.get('failed', False): output = [line.strip() for line in output['stdout_lines'][0] if len(line) > 0] for idx, line in enumerate(output): if end_line in line: @@ -413,7 +417,7 @@ def get_all_lldp_neighbor_details_for_port(self, physical_port): try: command = "show lldp neigh {} detail".format(physical_port) output = self.commands(commands=[command], module_ignore_errors=True) - if not output['failed']: + if not output.get('failed', False): logger.debug('cisco lldp output: %s' % (output)) return output['stdout_lines'][0] return "Failed to get lldp detail info for {} due to {}".format(physical_port, output) @@ -427,7 +431,7 @@ def get_platform_from_cli(self): try: command = "show version | i ^cisco | utility head -n 1" output = self.commands(commands=[command], module_ignore_errors=True) - if not output['failed']: + if not output.get('failed', False): logger.debug('cisco lldp output: %s' % (output)) return output['stdout_lines'][0][0].split()[1] return "Failed to get platform info due to {}".format(output) @@ -441,7 +445,7 @@ def get_version_from_cli(self): try: command = 'show version | in "Version :"' output = self.commands(commands=[command], module_ignore_errors=True) - if not output['failed']: + if not output.get('failed', False): logger.debug('cisco lldp output: %s' % (output)) return output['stdout'][0].split()[-1].strip() return "Failed to get version info due to {}".format(output) @@ -455,7 +459,7 @@ def get_chassis_id_from_cli(self): try: command = "show lldp | i Chassis ID:" output = self.commands(commands=[command], module_ignore_errors=True) - if not output['failed']: + if not output.get('failed', False): logger.debug('cisco lldp output: %s' % (output)) return output['stdout_lines'][0][0].split()[-1] return "Failed to get chassis id info due to {}".format(output) @@ -625,7 +629,7 @@ def get_all_interfaces_in_pc(self, pc_name): pc_name = self.convert_pc_to_be(pc_name) command = "show lacp {} | begin eceive".format(pc_name) output = self.commands(commands=[command], module_ignore_errors=True) - if not output["failed"]: + if not output.get("failed", False): logger.debug("cisco lldp output: %s" % (output)) interface = [ self.elongate_cisco_interface(line.split()[0]) @@ -651,7 +655,7 @@ def check_interface_status(self, interface): command = "show interfaces {}".format(pc_name) success_criteria = "line protocol is up" intf_status_output = self.commands(commands=[command], module_ignore_errors=True) - if not intf_status_output["failed"]: + if not intf_status_output.get("failed", False): logger.info("Interface status check: {} sent to {}".format(command, self.hostname)) is_up = success_criteria in intf_status_output["stdout"][0].lower() return is_up, intf_status_output["stdout_lines"][0] @@ -668,7 +672,7 @@ def get_isis_adjacency(self): logger.info("Gathering ISIS adjacency details") command = "show isis adjacency" output = self.commands(commands=[command], module_ignore_errors=True) - if not output["failed"]: + if not output.get("failed", False): for row in output["stdout_lines"][0][3:-2]: row = row.split() isis_details[row[0]] = dict() @@ -706,7 +710,7 @@ def get_isis_database(self, queue=None): command = "show isis database" output = self.commands(commands=[command], module_ignore_errors=True) lsp_entries = {} - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "*" in line: outline = line.replace("*", "").split() @@ -737,7 +741,7 @@ def get_bgp_status(self): try: output = self.commands(commands=[command], module_ignore_errors=True) bgp_status = {} - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0][1:]: line = line.strip().split() if line[-1].isdigit(): @@ -770,7 +774,7 @@ def get_bgp_session_status(self, peer_ip): """ bgp_peer_details = self.get_bgp_session_details(peer_ip) try: - if not bgp_peer_details["failed"]: + if not bgp_peer_details.get("failed", False): for line in bgp_peer_details["stdout_lines"][0]: if "BGP state" in line: bgp_session_status = line.strip().split()[3].strip(",") @@ -792,7 +796,7 @@ def is_prefix_advertised_to_peer(self, prefix, peer_ip): command = "show bgp advertised neighbor {} summary | in {}".format(peer_ip, prefix) output = self.commands(commands=[command], module_ignore_errors=True) prefix_adv_status = False - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if prefix in line: prefix_adv_status = True @@ -810,7 +814,7 @@ def get_ldp_oper_neighbor_ips(self): try: command = 'show mpls ldp neighbor | include "Peer LDP Identifier:|State:"' output = self.commands(commands=[command], module_ignore_errors=True) - if not output["failed"]: + if not output.get("failed", False): ldp_op_list = [] for idx in range(0, len(output["stdout_lines"][0]), 2): line1 = output["stdout_lines"][0][idx] @@ -830,7 +834,7 @@ def get_next_hop_physical_interface_list(self, destination_ip): command = "show route {} | include via".format(destination_ip) output = self.commands(commands=[command], module_ignore_errors=True) interface_list = [] - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "via" in line: next_hop = line.split("via")[-1].strip() @@ -873,7 +877,7 @@ def get_egress_interface_for_lsp(self, lsp_name): try: command = "show mpls traffic-eng tunnels name {} | include Hop0".format(lsp_name) output = self.commands(commands=[command], module_ignore_errors=True) - if not output["failed"]: + if not output.get("failed", False): list_of_interface = [] for line in output["stdout_lines"][0]: if "Hop0" in line: @@ -963,7 +967,7 @@ def set_rekey_period(self, profile_name, rekey_period_value): try: command = "macsec-policy {} sak-rekey-interval seconds {}".format(profile_name, rekey_period_value) output = self.config(lines=[command]) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -984,7 +988,7 @@ def get_macsec_profile(self, interface): macsec psk-keychain ptx10k-64hexCAK fallback-psk-keychain ptx10k-64hexCAK-fallback policy macsec-xpn-256 ! """ - if not output["failed"]: + if not output.get("failed", False): return True, output['stdout_lines'][0][1].split()[-1].strip() return False, output except Exception as e: @@ -1007,7 +1011,7 @@ def get_macsec_status_logs(self, interface, last_count="30", log_type="ESTABLISH return True, "log {} not supported on device {}".format(log_type, self.hostname) command = "show logging last {} | include {} | include {}".format(last_count, log_type, interface) output = self.commands(commands=[command])["stdout"][0] - if not output["failed"]: + if not output.get("failed", False): return len(output["stdout_lines"][0]) > 0, output["stdout"][0] return False, str(output) except Exception as e: @@ -1099,7 +1103,7 @@ def get_macsec_config(self, interface): command = "show running-config formal interface {} macsec psk-keychain".format(interface) output = self.commands(commands=[command]) # Returning only MACSEC config. - if not output["failed"]: + if not output.get("failed", False): for config in output["stdout_lines"][0]: if "psk" in config: return True, config @@ -1115,7 +1119,7 @@ def apply_macsec_interface_config(self, commands): """ try: output = self.config(lines=commands) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1129,7 +1133,7 @@ def delete_macsec_interface_config(self, interface): try: command = "no interface {} macsec ".format(interface) output = self.config(lines=command) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1186,7 +1190,7 @@ def remove_int_from_portchan(self, interface, pcnum): command = ["no bundle id"] parents = ["interface {}".format(interface)] output = self.config(lines=command, parents=parents) - if not output["failed"]: + if not output.get("failed", False): return True, "remove interface {} from ether-bundle {}".format(interface, pcnum) else: return False, "Failed to remove interface {} from ether-bundle {}".format(interface, pcnum) @@ -1205,7 +1209,7 @@ def put_int_in_portchan(self, interface, pcnum): try: command = ["interface {} bundle id {} mode active".format(interface, pcnum)] output = self.config(lines=command) - if not output["failed"]: + if not output.get("failed", False): return True, "Added interface {} from ether-bundle {}".format(interface, pcnum) else: return False, "Failed to add interface {} from ether-bundle {}".format(interface, pcnum) @@ -1224,7 +1228,7 @@ def run_configure_command_test(self): try: command = ["alias testversion show version"] output = self.config(lines=command) - if not output["failed"]: + if not output.get("failed", False): rollback_command = ["no alias testversion"] self.config(lines=rollback_command) return True, output @@ -1292,7 +1296,7 @@ def check_for_aggregate_route_generation(self, agg_prefix): command = "show route | include {}".format(agg_prefix) agg_route_gen_status = False output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if agg_prefix in line: agg_route_gen_status = True @@ -1305,7 +1309,7 @@ def get_list_of_location(self): command = "show platform | include NSHUT | include CPU | exclude RP" output = self.commands(commands=[command]) location_list = [] - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "CPU" in line: location_list.append(line.split()[0]) @@ -1316,7 +1320,7 @@ def get_ipfix_export_data_count(self, location): packets_exported = 0 command = 'show flow exporter IPFIX_MSAZ location {} | include "Packets exported:"'.format(location) output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "Packets exported:" in line: packets_exported += int(line.split()[2]) @@ -1350,7 +1354,7 @@ def apply_sample_filter_to_interface(self, filter_name, interface): interface, filter_name, filter_name ) output = self.config(lines=[command]) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1360,7 +1364,7 @@ def reboot_chassis(self): try: command = "admin hw-module location all reload noprompt" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: diff --git a/tests/common/devices/csonic.py b/tests/common/devices/csonic.py index e780f32ee75..33e8b738dfc 100644 --- a/tests/common/devices/csonic.py +++ b/tests/common/devices/csonic.py @@ -8,6 +8,7 @@ import json import logging +import os import subprocess from tests.common.devices.base import NeighborDevice @@ -104,6 +105,14 @@ def shell(self, cmd, **kwargs): """Run a shell command (compatible with Ansible shell module interface).""" return self._docker_exec(cmd, **kwargs) + def run_command(self, cmd, **kwargs): + """Run a single command inside the container and return its result.""" + return self._docker_exec(cmd, **kwargs) + + def run_command_list(self, cmds, **kwargs): + """Run a list of commands inside the container, returning a result per command.""" + return [self._docker_exec(cmd, **kwargs) for cmd in cmds] + def shutdown(self, ifname): """Shut down an interface.""" logger.info("CsonicHost [%s] shutting down %s", self.container_name, ifname) @@ -156,3 +165,52 @@ def config(self, lines=None, parents=None): vtysh_cmd += " -c '{}'".format(c) return self._docker_exec(vtysh_cmd) + + def fetch(self, src=None, dest=None, **kwargs): + """ + Copy a file out of the cSONiC container to the local controller, mimicking + the Ansible ``fetch`` module closely enough for techsupport collection. + + For a local VM host this uses ``docker cp``; for a remote VM host the file + is first copied to the host's /tmp via ``docker cp`` over SSH and then + pulled back with ``scp``. + """ + if not src or not dest: + return {'failed': True, 'msg': 'fetch requires src and dest'} + + dest_dir = os.path.join(dest, self.container_name) + os.makedirs(dest_dir, exist_ok=True) + local_dest = os.path.join(dest_dir, os.path.basename(src)) + + try: + if self.is_local: + cp_cmd = ['docker', 'cp', + '{}:{}'.format(self.container_name, src), local_dest] + result = subprocess.run(cp_cmd, capture_output=True, text=True, + timeout=kwargs.get('timeout', 120)) + else: + remote_tmp = '/tmp/{}_{}'.format(self.container_name, os.path.basename(src)) + ssh_base = ['ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null'] + target = self.vm_host_ip + if self.vm_host_user: + target = '{}@{}'.format(self.vm_host_user, self.vm_host_ip) + subprocess.run(ssh_base + [target, + 'docker cp {}:{} {}'.format(self.container_name, src, remote_tmp)], + capture_output=True, text=True, timeout=kwargs.get('timeout', 120)) + scp_base = ['scp', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', + '{}:{}'.format(target, remote_tmp), local_dest] + result = subprocess.run(scp_base, capture_output=True, text=True, + timeout=kwargs.get('timeout', 120)) + + rc = result.returncode + return { + 'failed': rc != 0, + 'rc': rc, + 'dest': local_dest, + 'stderr': result.stderr.strip(), + } + except subprocess.TimeoutExpired: + logger.error("CsonicHost [%s] fetch timed out: %s", self.container_name, src) + return {'failed': True, 'msg': 'fetch timed out', 'dest': local_dest} diff --git a/tests/common/devices/eos.py b/tests/common/devices/eos.py index ecdfaf87d6c..2bf8e959f7c 100644 --- a/tests/common/devices/eos.py +++ b/tests/common/devices/eos.py @@ -22,6 +22,226 @@ def _raise_err(msg): } +_INTF_TOKEN_RE = re.compile(r'(?<=\binterface\s)(\S+)|(?<=\binterfaces\s)(\S+)') + + +def _apply_intf_map(value, intf_map): + """Translate interface tokens via ``intf_map`` (converged-peer aware). + + On converged (multi-VRF) topologies a single cEOS VM hosts every logical + neighbor and the per-logical-neighbor interface name (e.g. ``Ethernet1`` + from minigraph) maps to a different physical interface on the shared VM + (e.g. ``Ethernet4``). ``ceos_topo_converger`` records that mapping in + ``intf_map`` (``{logical: converged}``). + + This rewrites ``interface `` / ``interfaces `` tokens inside + ``value`` (string, dict with ``command`` key, or list of either) so test + code can keep using the per-logical name from minigraph. Returns ``value`` + untouched when ``intf_map`` is empty (stock topology) so behavior is + byte-identical there. + """ + if not intf_map or value is None: + return value + + def _sub(match): + name = match.group(1) or match.group(2) + return intf_map.get(name, name) + + def _rewrite_one(item): + if isinstance(item, str): + return _INTF_TOKEN_RE.sub(_sub, item) + if isinstance(item, dict) and isinstance(item.get('command'), str): + new_item = dict(item) + new_item['command'] = _INTF_TOKEN_RE.sub(_sub, item['command']) + return new_item + return item + + if isinstance(value, list): + return [_rewrite_one(item) for item in value] + return _rewrite_one(value) + + +def _vrf_scope_bgp_parents(parents, vrf, prime_asn): + """Rewrite ``router bgp `` config parents to be VRF-scoped. + + On converged (multi-VRF) topologies a single cEOS VM hosts every logical + neighbor as a VRF under one global ``router bgp `` process. + Legacy test code targets ``router bgp `` in the default VRF, which on + such a VM lands in the wrong place. This rewrites those parents to + ``router bgp `` / ``vrf `` so existing test code works + unchanged. + + Returns ``parents`` untouched when no VRF scoping applies (vrf unset, no + ``router bgp`` parent, or the parents are already VRF-scoped). + """ + if not vrf or parents is None: + return parents + as_list = parents if isinstance(parents, list) else [parents] + if any(str(p).strip().startswith('vrf ') for p in as_list): + return parents + rewritten = [] + changed = False + for parent in as_list: + if str(parent).strip().startswith('router bgp'): + rewritten.append('router bgp {}'.format(prime_asn)) + rewritten.append('vrf {}'.format(vrf)) + changed = True + else: + rewritten.append(parent) + return rewritten if changed else parents + + +_BASH_PREFIX_RE = re.compile(r'^(\s*)bash\s+') +_BASH_ALREADY_SCOPED_RE = re.compile(r'^\s*bash\s+(?:sudo\s+)?ip\s+netns\s+exec\b') + + +def _vrf_scope_bash_commands(commands, vrf): + """Wrap ``bash `` invocations with ``sudo ip netns exec ns-``. + + On converged (multi-VRF) cEOS hosts BGP/data-plane routes live in the + per-VRF Linux network namespace ``ns-`` (Arista's standard mapping + between EOS VRFs and Linux namespaces). A plain ``bash `` invocation + via the EOS CLI runs in the *default* namespace which on converged has + no route to DUT data-plane IPs, so any network tool (snmpget, ping, + curl, traceroute, ...) fails with "Network is unreachable". + + This rewrites ``bash `` into + ``bash sudo ip netns exec ns- `` so the tool runs in the VRF + that carries the BGP-learned routes to the DUT. Native EOS CLI commands + (no ``bash`` prefix) and already-namespaced commands are left untouched. + Returns ``commands`` unchanged when no VRF is set (stock topology). + """ + if not vrf or commands is None: + return commands + scope_prefix = 'bash sudo ip netns exec ns-{} '.format(vrf) + + def _rewrite_text(text): + if not _BASH_PREFIX_RE.match(text): + return text + if _BASH_ALREADY_SCOPED_RE.match(text): + return text + return _BASH_PREFIX_RE.sub(scope_prefix, text, count=1) + + def _rewrite_one(item): + if isinstance(item, str): + return _rewrite_text(item) + if isinstance(item, dict) and isinstance(item.get('command'), str): + new_item = dict(item) + new_item['command'] = _rewrite_text(item['command']) + return new_item + return item + + if isinstance(commands, list): + return [_rewrite_one(c) for c in commands] + return _rewrite_one(commands) + + +_SHOW_BGP_RE = re.compile(r'^\s*show\s+(?:ip|ipv6)\s+bgp\b', re.IGNORECASE) + + +def _vrf_scope_eos_reads(commands, vrf): + """VRF-scope raw ``show ip|ipv6 bgp`` reads on converged hosts. + + Tests that read the BGP table with a plain + ``run_command('show ip bgp ...')`` hit the prime's *default* VRF, which on + a converged peer carries only the backplane -- the logical neighbor's + learned/advertised routes live under its per-neighbor VRF (same rationale + as ``get_route``). This injects ``vrf `` into such reads, before any + ``| `` pipe where EOS requires it (``show ip bgp ... vrf X`` is + valid, ``show ip bgp vrf X ...`` is not). + + Only plain string commands are rewritten; the structured ``dict`` commands + built by ``get_route``/``run_command_json`` are left untouched (``get_route`` + already scopes its own VRF). Commands that already carry an explicit + ``vrf`` token, and non ``show ... bgp`` commands (e.g. + ``show run | grep 'router bgp'``), are left untouched. Returns ``commands`` + unchanged when no VRF is set so stock topologies are byte-identical. + """ + if not vrf or commands is None: + return commands + + def _rewrite_text(text): + if not _SHOW_BGP_RE.match(text): + return text + if re.search(r'\bvrf\b', text, re.IGNORECASE): + return text + if '|' in text: + head, _, tail = text.partition('|') + return '{} vrf {} |{}'.format(head.rstrip(), vrf, tail) + return '{} vrf {}'.format(text.rstrip(), vrf) + + def _rewrite_one(item): + if isinstance(item, str): + return _rewrite_text(item) + return item + + if isinstance(commands, list): + return [_rewrite_one(c) for c in commands] + return _rewrite_one(commands) + + +# Loopback IDs created ad-hoc by tests via the lowercase +# ``interface loopback `` idiom collide with the converged prime's per-VRF +# ``Loopback`` interfaces (each merged sub-peer owns a small-numbered +# Loopback for its router-id). EOS interface names are global, so a test +# creating ``loopback 10`` would otherwise steal ``Loopback10`` from another +# VRF. On converged hosts we shift the test's loopback id by this offset into +# an unused range (well above the per-VRF loopback space, which is bounded by +# the cEOS interface limit) and place it in the neighbor's VRF. Validated on +# the cEOS lab image (``Loopback1010`` accepted). +_CONVERGED_TEST_LOOPBACK_OFFSET = 1000 + +_ROUTER_BGP_RE = re.compile(r'^\s*router\s+bgp\s+\d+', re.IGNORECASE) +# Match only the lowercase ad-hoc idiom (``interface loopback ``); the +# converged prime's real interfaces render as capitalized ``Loopback`` and +# must not be renumbered. +_TEST_LOOPBACK_RE = re.compile(r'^(\s*)interface\s+loopback\s+(\d+)\s*$') + + +def _vrf_scope_eos_config(commands, vrf, prime_asn): + """VRF-scope ad-hoc BGP/loopback config pushed via ``run_command_list``. + + Some tests push config by feeding ``eos_command`` a ``configure``-prefixed + list of native CLI lines (instead of ``eos_config``). On converged hosts + that config must be VRF-scoped exactly like ``eos_config`` does: + + * ``router bgp `` -> ``router bgp `` followed by + ``vrf `` so the nested ``address-family``/``network`` statements + land in the neighbor's VRF rather than the prime's default process. + * ``interface loopback `` -> a collision-free + ``Loopback`` (see ``_CONVERGED_TEST_LOOPBACK_OFFSET``) + followed by ``vrf ``, so the connected host route used to source an + advertised ``network`` exists in the right VRF instead of clobbering a + sub-peer's Loopback. + + Returns ``commands`` unchanged when no VRF is set, the input is not a list, + or nothing matches -- so stock topologies and ordinary command lists are + byte-identical. + """ + if not vrf or not isinstance(commands, list): + return commands + if not any(isinstance(c, str) + and (_ROUTER_BGP_RE.match(c) or _TEST_LOOPBACK_RE.match(c)) + for c in commands): + return commands + rewritten = [] + for cmd in commands: + if isinstance(cmd, str): + loopback_match = _TEST_LOOPBACK_RE.match(cmd) + if loopback_match: + indent, num = loopback_match.group(1), int(loopback_match.group(2)) + rewritten.append('{}interface Loopback{}'.format( + indent, num + _CONVERGED_TEST_LOOPBACK_OFFSET)) + rewritten.append('vrf {}'.format(vrf)) + continue + if _ROUTER_BGP_RE.match(cmd) and prime_asn: + rewritten.append('router bgp {}'.format(prime_asn)) + rewritten.append('vrf {}'.format(vrf)) + continue + rewritten.append(cmd) + return rewritten + + class EosHost(AnsibleHostBase): """ @summary: Class for Eos switch @@ -47,6 +267,17 @@ def __init__(self, ansible_adhoc, hostname, eos_user, eos_passwd, self.shell_user = shell_user self.shell_passwd = shell_passwd self.is_multi_asic = False + # VRF scoping for converged (multi-VRF) topologies. When set, BGP config + # parents are transparently rewritten to be VRF-scoped in eos_config(). + # Left as None on stock topologies so behavior is byte-identical. + self.bgp_vrf = None + self.bgp_prime_asn = None + # Interface-name translation for converged topologies. When set, any + # ``interface `` / ``interfaces `` token in eos_config() + # parents/lines and eos_command() commands is translated through this + # ``{logical: converged}`` map so tests can keep using the per-logical + # name reported by minigraph. Left as None on stock topologies. + self.intf_map = None AnsibleHostBase.__init__(self, ansible_adhoc, hostname) self.localhost = ansible_adhoc(inventory='localhost', connection='local', host_pattern="localhost")["localhost"] @@ -83,6 +314,58 @@ def __str__(self): def __repr__(self): return self.__str__() + def eos_config(self, *args, **kwargs): + """VRF-aware wrapper around the ``eos_config`` Ansible module. + + All EosHost config writes (config(), shutdown(), no_shutdown_bgp(), + and direct test calls) funnel through here. On converged topologies + BGP config parents are transparently VRF-scoped and per-logical + interface names are translated to the converged VM's actual interface + names; on stock topologies the call is passed through unchanged. + """ + if self.intf_map: + for key in ('parents', 'lines'): + if key in kwargs: + kwargs[key] = _apply_intf_map(kwargs[key], self.intf_map) + if 'parents' in kwargs: + kwargs['parents'] = _vrf_scope_bgp_parents( + kwargs['parents'], self.bgp_vrf, self.bgp_prime_asn) + ansible_eos_config = self.__getattr__('eos_config') + return ansible_eos_config(*args, **kwargs) + + def eos_command(self, *args, **kwargs): + """Converged-peer-aware wrapper around the ``eos_command`` Ansible module. + + On converged topologies these transparent rewrites happen: + + * ``interface `` / ``interfaces `` tokens in ``commands`` + are translated through ``intf_map`` so tests can keep using the + per-logical-neighbor name from minigraph. + * ``bash `` invocations are wrapped with + ``bash sudo ip netns exec ns- `` so network tools + (snmpget, ping, ...) execute in the VRF that holds the BGP routes + to the DUT instead of the route-less default namespace. + * raw ``show ip|ipv6 bgp ...`` reads are VRF-scoped (``vrf `` + injected before any ``| `` pipe) so they read the logical + neighbor's routes instead of the prime's default VRF. + * ad-hoc ``router bgp``/``interface loopback`` config lines pushed + through ``run_command_list`` are VRF-scoped and de-collided the same + way ``eos_config`` scopes BGP config (see ``_vrf_scope_eos_config``). + + On stock topologies the call is passed through unchanged. + """ + if self.intf_map and 'commands' in kwargs: + kwargs['commands'] = _apply_intf_map(kwargs['commands'], self.intf_map) + if self.bgp_vrf and 'commands' in kwargs: + kwargs['commands'] = _vrf_scope_bash_commands( + kwargs['commands'], self.bgp_vrf) + kwargs['commands'] = _vrf_scope_eos_reads( + kwargs['commands'], self.bgp_vrf) + kwargs['commands'] = _vrf_scope_eos_config( + kwargs['commands'], self.bgp_vrf, self.bgp_prime_asn) + ansible_eos_command = self.__getattr__('eos_command') + return ansible_eos_command(*args, **kwargs) + @retry(RunAnsibleModuleFail, tries=3, delay=5) def shutdown(self, interface_name): out = self.eos_config( @@ -202,7 +485,7 @@ def set_interface_lacp_rate_mode(self, interface_name, mode): # FIXME: out['failed'] will be False even when a command is deprecated, so we have to check out['changed'] # However, if the lacp rate is already in expected state, out['changed'] will be False and treated as # error. - if out['failed'] is True or out['changed'] is False: + if out.get('failed', False) is True or out['changed'] is False: # new eos deprecate lacp rate and use lacp timer command out = self.eos_config( lines=['lacp timer %s' % mode], @@ -325,12 +608,38 @@ def exec_template(self, ansible_root, ansible_playbook, inventory, **kwargs): def get_route(self, prefix, vrf=None): cmd = 'show ip bgp' if ipaddress.ip_network(prefix.encode().decode()).version == 4 else 'show ipv6 bgp' cmd = '{} {}'.format(cmd, prefix) - if vrf: - cmd = '{} vrf {}'.format(cmd, vrf) - return self.eos_command(commands=[{ + # In converged (multi-VRF) mode, routes for this logical neighbor + # live under the per-neighbor VRF on the prime EOS peer (``self.bgp_vrf``). + # When the caller passes no explicit ``vrf=``, auto-scope to + # ``self.bgp_vrf`` and surface the returned ``vrfs/`` entry + # under ``vrfs/default`` so existing readers that hardcode 'default' + # (e.g. tests/bgp/test_bgp_bbr.py, tests/filterleaf/filterleaf_helpers.py, + # tests/vlan/test_vlan_ports_down.py) keep working. We alias rather + # than rename so the original VRF key is still present for any caller + # that iterates ``vrfs.keys()``. Callers that pass an explicit + # ``vrf=`` (e.g. tests/bgp/bgp_helpers.py) get the response unmodified. + # On stock topologies ``self.bgp_vrf`` is None and behavior is + # byte-identical. + effective_vrf = vrf + alias_as_default = False + if effective_vrf is None and self.bgp_vrf: + effective_vrf = self.bgp_vrf + alias_as_default = True + if effective_vrf: + cmd = '{} vrf {}'.format(cmd, effective_vrf) + out = self.eos_command(commands=[{ 'command': cmd, 'output': 'json' }])['stdout'][0] + if alias_as_default and isinstance(out, dict) and isinstance(out.get('vrfs'), dict): + vrfs = out['vrfs'] + if effective_vrf in vrfs: + # Overwrite any existing 'default' entry: in converged mode the + # actual default VRF on the prime carries the backplane + # config, not this logical neighbor's routes, so legacy callers + # expect the per-neighbor view here. + vrfs['default'] = vrfs[effective_vrf] + return out def run_command_json(self, cmd): return self.eos_command(commands=[{ @@ -596,7 +905,7 @@ def set_interface_lacp_time_multiplier(self, interface_name, multiplier): lines=['lacp timer multiplier %d' % multiplier], parents='interface %s' % interface_name) - if out['failed'] is True or out['changed'] is False: + if out.get('failed', False) is True or out['changed'] is False: logging.warning("Unable to set interface [%s] lacp timer multiplier to [%d]" % (interface_name, multiplier)) else: logging.info("Set interface [%s] lacp timer to [%d]" % (interface_name, multiplier)) diff --git a/tests/common/devices/juniper.py b/tests/common/devices/juniper.py index 70dd0c4e0f8..b26e8f2e293 100644 --- a/tests/common/devices/juniper.py +++ b/tests/common/devices/juniper.py @@ -239,7 +239,7 @@ def get_prefix_nh_info(self, prefix, table="inet.0"): nh_result = [] command = "show route {} table {} active-path exact".format(prefix, table) json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): prefix_info = json_output["stdout"][0]["route-information"][0]["route-table"][0]["rt"][0] if prefix in prefix_info["rt-destination"][0]["data"]: nh_info = {"nh_ip": None, "nh_interface": None, "nh_label_info": None, "nh_lsp_name": None} @@ -298,7 +298,7 @@ def get_list_of_fpc_numbers(self): command = "show chassis hardware | match fpc | except CPU" output = self.commands(session_type="network_cli", commands=[command], display="text") fpc_list = [] - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "FPC" in line: fpc_list.append(line.split()[1]) @@ -309,7 +309,7 @@ def get_ipfix_export_data_count(self, fpc_no): ipfix_export_data = {"count": 0} command = "show services accounting flow inline-jflow fpc-slot {}".format(fpc_no) output = self.commands(commands=[command], display="json") - if not output["failed"]: + if not output.get("failed", False): ipfix_export_data["count"] = output["stdout"][0]["services-accounting-information"][0][ "inline-jflow-flow-information" ][0]["inline-flows-exported"][0]["data"] @@ -347,7 +347,7 @@ def apply_sample_filter_to_interface(self, filter_name, interface): configs = [] configs.append("set interfaces {} unit 0 family inet filter input {}".format(interface, filter_name)) output = self.config(lines=configs, comment="push ipfix configs") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -585,7 +585,7 @@ def get_isis_adjacency(self): command = "show isis adjacency" json_output = self.commands(commands=[command], display="json") isis_adj_info = {} - if not json_output["failed"]: + if not json_output.get("failed", False): for line in json_output["stdout"][0]["isis-adjacency-information"][0]["isis-adjacency"]: isis_adj_info.update( { @@ -621,7 +621,7 @@ def get_isis_database(self, queue=None): command = "show isis database" json_output = self.commands(commands=[command], display="json") lsp_entries = {} - if not json_output["failed"]: + if not json_output.get("failed", False): for entry in json_output["stdout"][0]["isis-database-information"][0]["isis-database"][1][ "isis-database-entry" ]: @@ -740,7 +740,7 @@ def get_lldp_neighbors(self): try: command = "show lldp neighbors" output = self.commands(commands=[command], display="json") - if not output["failed"]: + if not output.get("failed", False): for lines in output["stdout"][0]["lldp-neighbors-information"][0]["lldp-neighbor-information"]: lldp_details.update( { @@ -794,7 +794,7 @@ def check_interface_status(self, interface): command = "show interfaces {}".format(pc_name) success_criteria = "physical link is up" intf_status_output = self.commands(session_type="network_cli", commands=[command], display="text") - if not intf_status_output["failed"]: + if not intf_status_output.get("failed", False): logger.info("Interface status check: {} sent to {}".format(command, self.hostname)) is_up = success_criteria in intf_status_output["stdout"][0].lower() return is_up, intf_status_output["stdout"][0] @@ -814,7 +814,7 @@ def get_all_interfaces_in_pc(self, pc_name): command = "show lacp interfaces {}".format(pc_name) try: json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): interfaces_data = self.extract_val_from_json(json_output["stdout"][0], "lag-lacp-protocol")[0] interfaces = [line["name"][0]["data"] for line in interfaces_data] return interfaces @@ -830,7 +830,7 @@ def get_all_lldp_neighbor_details_for_port(self, physical_port): try: command = "show lldp neighbor interface {}".format(physical_port) json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): return json_output["stdout"][0] return json_output except Exception as e: @@ -844,7 +844,7 @@ def get_chassis_id_from_cli(self): command = "show lldp local-information" try: json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): chassis_id = json_output["stdout"][0]["lldp-local-info"][0]["lldp-local-chassis-id"][0]["data"] return chassis_id return "Failed to get chassis info due to {}".format(json_output) @@ -860,7 +860,7 @@ def get_mgmt_ip_from_cli(self): try: command = "show lldp local-information" json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): mgmt_ip = json_output["stdout"][0]["lldp-local-info"][0]["lldp-local-management-address-address"][0][ "data" ] @@ -876,7 +876,7 @@ def get_platform_from_cli(self): try: command = "show lldp local-information" json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): system_description = json_output["stdout"][0]["lldp-local-info"][0]["lldp-local-system-description"][ 0 ]["data"] @@ -894,7 +894,7 @@ def get_version_from_cli(self): try: command = "show version" json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): version = json_output["stdout"][0]["software-information"][0]["junos-version"][0]["data"] return version return "Failed to get version due to {}".format(json_output) @@ -1034,7 +1034,7 @@ def get_macsec_status_logs(self, interface, last_count="30", log_type="ESTABLISH last_count, log_type ) output = self.commands(session_type="network_cli", commands=[command], display="text") - if not output["failed"]: + if not output.get("failed", False): return len(output["stdout_lines"][0]) > 0, output["stdout"][0] return False, "Failed to get macsec status logs due to {}".format(output) except Exception as e: @@ -1054,7 +1054,7 @@ def get_macsec_profile(self, interface): example of output: set security macsec interfaces et-2/0/14 connectivity-association macsec-xpn-256-ae16 """ - if not output["failed"]: + if not output.get("failed", False): return True, output["stdout"][0].split()[-1] return False, "Failed to get macsec profile due to {}".format(output) except Exception as e: @@ -1090,7 +1090,7 @@ def set_macsec_key(self, profile_name, key, key_type, interface): test_msg = "Key type {} not supported".format(key_type) output = {"failed": True, "msg": test_msg} - if not output["failed"]: + if not output.get("failed", False): return True, str(output) return False, "Failed to set macsec key due to {}".format(output) except Exception as e: @@ -1133,7 +1133,7 @@ def get_macsec_interface_statistics(self, interface): ... Output trimmed """ - if not json_output["failed"]: + if not json_output.get("failed", False): sc_dict_out = json_output["stdout"][0]["macsec-statistics"][0]["secure-channel-received"][0] validated_bytes = sc_dict_out["validated-bytes"][0]["data"] decrypted_bytes = sc_dict_out["decrypted-bytes"][0]["data"] @@ -1166,7 +1166,7 @@ def set_deactivate_macsec_interface(self, interfaces): commands.append("deactivate security macsec interfaces {}".format(interface)) try: output = self.config(lines=commands, comment="deactivate macsec interface") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to deactivate macsec interface due to {}".format(output) except Exception as e: @@ -1182,7 +1182,7 @@ def set_activate_macsec_interface(self, interfaces): commands.append("activate security macsec interfaces {}".format(interface)) try: output = self.config(lines=commands, comment="activate macsec interface") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to activate macsec interface due to {}".format(output) except Exception as e: @@ -1202,7 +1202,7 @@ def get_macsec_config(self, interface): example of output: set security macsec interfaces et-2/0/14 connectivity-association macsec-xpn-256-ae16 """ - if not output["failed"]: + if not output.get("failed", False): # strip to remove extra /n with string return True, output["stdout"][0].strip() return False, "Failed to get macsec config due to {}".format(output) @@ -1216,7 +1216,7 @@ def apply_macsec_interface_config(self, commands): """ output = self.config(lines=commands, comment="apply_macsec_interface_config") try: - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to apply macsec interface config due to {}".format(output) except Exception as e: @@ -1230,7 +1230,7 @@ def delete_macsec_interface_config(self, interface): command = "delete security macsec interfaces {}".format(interface) try: output = self.config(lines=[command], comment="remove MACSEC from physical interface") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to delete macsec interface config due to {}".format(output) except Exception as e: @@ -1247,7 +1247,7 @@ def set_rekey_period(self, profile_name, rekey_period_value): profile_name, rekey_period_value ) output = self.config(lines=[command], comment="rekey macsec interval") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to set rekey period due to {}".format(output) except Exception as e: @@ -1265,7 +1265,7 @@ def run_configure_command_test(self): try: command = "set system location building microsoft" output = self.config(lines=[command], comment="test configure command") - if not output["failed"]: + if not output.get("failed", False): rollback_command = "del system location building microsoft" self.config(lines=[rollback_command], comment="rollback test configure command") return True, output @@ -1280,7 +1280,7 @@ def pull_tacplus_source_address(self): try: command = "show configuration system tacplus-server | display set | match source-address" output = self.commands(session_type="network_cli", commands=[command], display="text") - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if "source-address" in line: return line.split()[-1] @@ -1369,7 +1369,7 @@ def apply_prod_tacacs(self, prod_tacacsserver, tacacs_secret, accounting_secret, ) try: output = self.config(lines=prod_configs, confirm=2) - if not output["failed"]: + if not output.get("failed", False): return True, "prod configs pushed to lab router, will rollback by itself in 2 minutes" return False, "Failed to apply prod tacacs config due to {}".format(output) except Exception as e: @@ -1410,7 +1410,7 @@ def get_bgp_status(self): try: command = "show bgp summary" json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): json_output_dict = json_output["stdout"][0]["bgp-information"][0]["bgp-peer"] bgp_status = {} for bgp_speak in json_output_dict: @@ -1440,7 +1440,7 @@ def get_bgp_session_status(self, peer_ip): """ try: bgp_peer_details = self.get_bgp_session_details(peer_ip) - if not bgp_peer_details["failed"]: + if not bgp_peer_details.get("failed", False): session_status = bgp_peer_details["stdout"][0]["bgp-information"][0]["bgp-peer"][0]["peer-state"][0][ "data" ] @@ -1458,7 +1458,7 @@ def check_for_aggregate_route_generation(self, agg_prefix): command = "show route protocol aggregate {} exact".format(agg_prefix) agg_route_gen_status = False output = self.commands(session_type="network_cli", commands=[command], display="text") - if not output["failed"]: + if not output.get("failed", False): for line in output["stdout_lines"][0]: if agg_prefix in line: agg_route_gen_status = True @@ -1477,7 +1477,7 @@ def is_prefix_advertised_to_peer(self, prefix, peer_ip): command = "show route advertising-protocol bgp {} {} exact".format(peer_ip, prefix) json_output = self.commands(commands=[command], display="json") prefix_adv_status = False - if not json_output["failed"]: + if not json_output.get("failed", False): route_table = json_output["stdout"][0]["route-information"][0]["route-table"][0]["rt"] for element in route_table: if "rt-destination" in element: @@ -1502,7 +1502,7 @@ def deactivate_bgp_with_ser(self): "deactivate protocols bgp group IPV6-ICR-SWAN", ] output = self.config(lines=commands, comment="deactivate bgp with ser") - if not output["failed"]: + if not output.get("failed", False): return True return False except Exception as e: @@ -1522,7 +1522,7 @@ def activate_bgp_with_ser(self): "activate protocols bgp group IPV6-ICR-SWAN", ] output = self.config(lines=commands, comment="activate bgp with ser") - if not output["failed"]: + if not output.get("failed", False): return True return False except Exception as e: @@ -1536,7 +1536,7 @@ def deactivate_protocol_rsvp(self): """ try: output = self.config(lines=["deactivate protocols rsvp"], comment="deactivate protocol rsvp") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, "Failed to deactivate rsvp protocol due to {}".format(output) except Exception as e: @@ -1550,7 +1550,7 @@ def activate_protocol_rsvp(self): """ try: output = self.config(lines=["activate protocols rsvp"], comment="activate protocol rsvp") - if not output["failed"]: + if not output.get("failed", False): return True, output else: return False, "Failed to activate rsvp protocol due to {}".format(output) @@ -1562,7 +1562,7 @@ def get_active_route_details(self, prefix_with_mask, table): try: command = "show route {} table {} active-path exact".format(prefix_with_mask, table) json_output = self.commands(commands=[command], display="json") - if not json_output["failed"]: + if not json_output.get("failed", False): if "route-table" in json_output["stdout"][0]["route-information"][0]: route_table = json_output["stdout"][0]["route-information"][0]["route-table"][0] destination = route_table["rt"][0]["rt-destination"][0]["data"] @@ -1585,7 +1585,7 @@ def shutdown_swan_agent(self): "deactivate system extensions extension-service application file apcacertagent-junos", ] output = self.config(lines=commands, comment="deactivate swan agent") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1602,7 +1602,7 @@ def enable_swan_agent(self): "activate system extensions extension-service application file apcacertagent-junos", ] output = self.config(lines=commands, comment="activate swan agent") - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: @@ -1662,7 +1662,7 @@ def check_rsvp_nbr(self, neighbor): ... Output trimmed """ - if not json_output["failed"]: + if not json_output.get("failed", False): rsvp_nbrs = json_output["stdout"][0]["rsvp-neighbor-information"][0]["rsvp-neighbor"] for rsvp_nbr in rsvp_nbrs: if str(rsvp_nbr["rsvp-neighbor-address"][0]["data"]) == str(neighbor): @@ -1686,7 +1686,7 @@ def get_loopback_ipv4_addr(self): command = "show configuration interfaces lo0" try: json_output = self.commands(commands=[command], display="json")["stdout"][0] - if not json_output["failed"]: + if not json_output.get("failed", False): ipv4_addresses = json_output["configuration"]["interfaces"]["interface"][0]["unit"][0]["family"][ "inet" ]["address"] @@ -1710,7 +1710,7 @@ def remove_int_from_portchan(self, interface, pcnum): try: commands = ["deactivate interfaces {} gigether-options 802.3ad".format(interface)] output = self.config(lines=commands, comment="deactivate interface {} from ae {}".format(interface, pcnum)) - if not output["failed"]: + if not output.get("failed", False): return True, "Deactivated interface {} from ae{}".format(interface, pcnum) return False, "Failed to deactivate interface {} from ae{}".format(interface, pcnum) except Exception as e: @@ -1726,7 +1726,7 @@ def put_int_in_portchan(self, interface, pcnum): try: commands = ["activate interfaces {} gigether-options 802.3ad".format(interface)] output = self.config(lines=commands, comment="activate interface {} from ae {}".format(interface, pcnum)) - if not output["failed"]: + if not output.get("failed", False): return True, "Activated interface {} from ae{}".format(interface, pcnum) return False, "Failed to activate interface {} from ae{}".format(interface, pcnum) except Exception as e: @@ -1736,7 +1736,7 @@ def reboot_chassis(self): try: command = "request system reboot both-routing-engines" output = self.commands(commands=[command]) - if not output["failed"]: + if not output.get("failed", False): return True, output return False, output except Exception as e: diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index 58b9c59fc0f..7833d48be41 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -458,6 +458,38 @@ def get_monit_services_status(self): return monit_services_status + def _retry_if_oci_exec_race(self, cmd, result, attempts=3, delay=2): + """ + Re-run `cmd` if `result` is a transient `docker exec` runc-setns race + (rc=127 with "OCI runtime exec failed: ... setns process | fork/exec + /proc/self/fd"). The OCI message can land on either stdout or stderr + depending on the docker / runc version, so we inspect both with a + DOTALL match so multi-line variants are still caught. The container + is actually running while this race fires, so a small bounded retry + keeps the framework from declaring the service unhealthy on what is + purely an exec infrastructure flake. Pass-through otherwise. + """ + pattern = r"OCI runtime exec failed:.*(starting setns process|fork/exec /proc/self/fd)" + + def _is_oci(res): + if not (isinstance(res, dict) and res.get("rc") == 127): + return False + output = (res.get("stdout") or "") + "\n" + (res.get("stderr") or "") + return bool(re.search(pattern, output, flags=re.DOTALL)) + + if not _is_oci(result): + return result + for attempt in range(1, attempts + 1): + time.sleep(delay) + retry = self.shell(cmd, module_ignore_errors=True) + if not _is_oci(retry): + logging.info( + "Recovered from transient docker exec OCI race on attempt %d for cmd: %s", + attempt, cmd, + ) + return retry + return result + def get_critical_group_and_process_lists(self, container_name): """ @summary: Get critical group and process lists by parsing the @@ -468,8 +500,10 @@ def get_critical_group_and_process_lists(self, container_name): critical_process_list = [] succeeded = True - file_content = self.shell("docker exec {} bash -c '[ -f /etc/supervisor/critical_processes ] \ - && cat /etc/supervisor/critical_processes'".format(container_name), module_ignore_errors=True) + cmd = "docker exec {} bash -c '[ -f /etc/supervisor/critical_processes ] \ + && cat /etc/supervisor/critical_processes'".format(container_name) + file_content = self.shell(cmd, module_ignore_errors=True) + file_content = self._retry_if_oci_exec_race(cmd, file_content) for line in file_content["stdout_lines"]: line_info = line.strip().split(':') if len(line_info) != 2: @@ -525,6 +559,10 @@ def critical_group_process(self): cmds.append(cmd) results = self.shell_cmds(cmds=cmds, continue_on_fail=True, module_ignore_errors=True, timeout=30)['results'] + # Re-run any commands hit by the transient `docker exec` runc-setns race. + # The target container is still running; only the new exec failed. + results = [self._retry_if_oci_exec_race(res['cmd'], res) for res in results] + # Extract service name of each command result, transform results list to a dict keyed by service name service_results = {} for res in results: @@ -613,6 +651,12 @@ def all_critical_process_status(self): cmds.append(cmd) results = self.shell_cmds(cmds=cmds, continue_on_fail=True, module_ignore_errors=True, timeout=60)['results'] + # Re-run any commands hit by the transient `docker exec` runc-setns race + # before we let the result feed parse_service_status_and_critical_process, + # which would otherwise see one garbage "OCI runtime exec failed..." line + # and flip the service to status=False even though the container is up. + results = [self._retry_if_oci_exec_race(res['cmd'], res) for res in results] + # Extract service name of each command result, transform results list to a dict keyed by service name service_results = {} for res in results: @@ -2338,7 +2382,7 @@ def ping_v4(self, ipv4, count=1, ns_arg=""): )) except RunAnsibleModuleFail: return False - return not rc['failed'] + return not rc.get('failed', False) def ping_v6(self, ipv6, count=1, ns_arg=""): """ @@ -2364,7 +2408,7 @@ def ping_v6(self, ipv6, count=1, ns_arg=""): )) except RunAnsibleModuleFail: return False - return not rc['failed'] + return not rc.get('failed', False) def is_backend_portchannel(self, port_channel, mg_facts): ports = mg_facts["minigraph_portchannels"].get(port_channel) @@ -3009,7 +3053,7 @@ def set_loopback(self, port: int, baud_rate: int = 9600, flow_control: bool = Fa ) res: ShellResult = self.shell(command, module_ignore_errors=True) - if res['failed']: + if res.get('failed', False): error_msg = f"Failed to start socat on port {port}: {res.get('stderr', '')}" logging.error(error_msg) raise RuntimeError(error_msg) @@ -3090,7 +3134,7 @@ def bridge(self, port1: int, port2: int, baud_rate: int = 9600, flow_control: bo ) res: ShellResult = self.shell(command, module_ignore_errors=True) - if res['failed']: + if res.get('failed', False): error_msg = f"Failed to bridge ports {port1} and {port2}: {res.get('stderr', '')}" logging.error(error_msg) raise RuntimeError(error_msg) @@ -3184,7 +3228,7 @@ def bridge_remote( ) res: ShellResult = self.shell(command, module_ignore_errors=True) - if res['failed']: + if res.get('failed', False): error_msg = f"Failed to bridge port {port} to {remote_host}:{remote_port}: {res.get('stderr', '')}" logging.error(error_msg) raise RuntimeError(error_msg) diff --git a/tests/common/devices/sonic_asic.py b/tests/common/devices/sonic_asic.py index 108e6440978..9c9a43acf59 100644 --- a/tests/common/devices/sonic_asic.py +++ b/tests/common/devices/sonic_asic.py @@ -294,7 +294,7 @@ def ping_v4(self, ipv4, count=1): )) except RunAnsibleModuleFail: return False - return not rc['failed'] + return not rc.get('failed', False) def ping_v6(self, ipv6, count=1): """ @@ -316,7 +316,7 @@ def ping_v6(self, ipv6, count=1): )) except RunAnsibleModuleFail: return False - return not rc['failed'] + return not rc.get('failed', False) def is_backend_portchannel(self, port_channel): mg_facts = self.sonichost.minigraph_facts(host=self.sonichost.hostname)['ansible_facts'] diff --git a/tests/common/helpers/gnmi_utils.py b/tests/common/helpers/gnmi_utils.py index 525e6fff810..9b215654a52 100644 --- a/tests/common/helpers/gnmi_utils.py +++ b/tests/common/helpers/gnmi_utils.py @@ -295,12 +295,12 @@ def create_revoked_cert_and_crl(localhost, ptfhost, duthost=None): localhost.shell(local_command) -def create_gnmi_certs(duthost, localhost, ptfhost): +def create_gnmi_certs(duthost, localhost, ptfhost, dut_ip=None): ''' Create GNMI client certificates ''' prepare_root_cert(localhost) - prepare_server_cert(duthost, localhost) + prepare_server_cert(duthost, localhost, dut_ip=dut_ip) prepare_client_cert(localhost) create_revoked_cert_and_crl(localhost, ptfhost) copy_certificate_to_dut(duthost) @@ -351,10 +351,10 @@ def create_root_cert(localhost, days): _write_pem_certificate("gnmiCA.pem", cert) -def prepare_server_cert(duthost, localhost, days="825"): +def prepare_server_cert(duthost, localhost, days="825", dut_ip=None): create_server_key(localhost) create_server_csr(localhost) - sign_server_certificate(duthost, localhost, days) + sign_server_certificate(duthost, localhost, days, dut_ip=dut_ip) def create_server_key(localhost): @@ -371,15 +371,20 @@ def create_server_csr(localhost): localhost.shell(local_command) -def sign_server_certificate(duthost, localhost, days): - """Sign gnmiserver.csr with the CA, backdated, with SAN (hostname.com + DUT mgmt IP).""" +def sign_server_certificate(duthost, localhost, days, dut_ip=None): + """Sign gnmiserver.csr with the CA, backdated, with SAN (hostname.com + DUT mgmt IP). + + When dut_ip is provided, it is used as the SAN IP address instead of + duthost.mgmt_ip. This lets callers bind the cert to the address the + gnmi server is actually reachable at (e.g. when bound to a non-default VRF). + """ ca_cert = _load_pem_certificate("gnmiCA.pem") ca_key = _load_pem_private_key("gnmiCA.key") csr = _load_pem_csr("gnmiserver.csr") not_before, not_after = _cert_validity_period(days) san_entries = [ x509.DNSName("hostname.com"), - x509.IPAddress(ipaddress.ip_address(duthost.mgmt_ip)), + x509.IPAddress(ipaddress.ip_address(dut_ip or duthost.mgmt_ip)), ] cert = ( x509.CertificateBuilder() diff --git a/tests/common/helpers/liquid_leakage_control_test_helper.py b/tests/common/helpers/liquid_leakage_control_test_helper.py index d20d231a6bd..b071f0ba10e 100644 --- a/tests/common/helpers/liquid_leakage_control_test_helper.py +++ b/tests/common/helpers/liquid_leakage_control_test_helper.py @@ -60,7 +60,7 @@ def get_leakage_status(dut): :param dut: DUT object representing a SONiC switch under test. :return: The leakage status of the DUT. """ - return dut.show_and_parse("show platform leakage status") + return dut.show_and_parse("show platform leak status") def get_leakage_status_in_health_system(dut): diff --git a/tests/common/helpers/monit.py b/tests/common/helpers/monit.py index 20c5245869b..a22d1ec3514 100644 --- a/tests/common/helpers/monit.py +++ b/tests/common/helpers/monit.py @@ -12,3 +12,4 @@ def check_monit_expected_container_logging(duthost): syslog_output = duthost.command("sudo grep 'ERR monit' /var/log/syslog")["stdout"] pytest_assert("Expected containers not running" not in syslog_output, f"Expected containers not running found in syslog. Output was:\n{syslog_output}") + return True diff --git a/tests/common/helpers/pfc_counters.py b/tests/common/helpers/pfc_counters.py index ce46e86617b..dea1114e0b5 100644 --- a/tests/common/helpers/pfc_counters.py +++ b/tests/common/helpers/pfc_counters.py @@ -168,6 +168,12 @@ def run_test(fanouthosts, duthost, conn_graph_facts, enum_fanout_graph_facts, le ).format(len(failures)) else: + """ Poll interval and timeout for waiting on counter updates """ + POLL_INTERVAL = 0.5 + POLL_TIMEOUT = 10 + """ Retry sending frames once if the counter does not update in time """ + MAX_RETRIES = 2 + for intf in active_phy_intfs: """only check priority 3 and 4: lossless priorities""" for priority in range(3, 5): @@ -196,16 +202,38 @@ def run_test(fanouthosts, duthost, conn_graph_facts, enum_fanout_graph_facts, le cmd = 'docker exec %s "python %s -i %s -p %d -t %d -n %d"' % ( onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) - peerdev_ans.host.config(cmd) + send_frames = lambda: peerdev_ans.host.config(cmd) # noqa: E731 else: cmd = "sudo python %s -i %s -p %d -t %d -n %d" % ( PFC_GEN_FILE_DEST, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) - peerdev_ans.host.command(cmd) + send_frames = lambda: peerdev_ans.host.command(cmd) # noqa: E731 + + send_frames() - time.sleep(5) + pfc_rx = {} + for attempt in range(1, MAX_RETRIES + 1): + """ Poll until counter reaches PKT_COUNT or timeout """ + deadline = time.time() + POLL_TIMEOUT + pfc_rx = duthost.sonic_pfc_counters(method="get")['ansible_facts'] + while pfc_rx[intf]['Rx'][priority] != str(PKT_COUNT) and time.time() < deadline: + time.sleep(POLL_INTERVAL) + pfc_rx = duthost.sonic_pfc_counters(method="get")['ansible_facts'] + + if pfc_rx[intf]['Rx'][priority] == str(PKT_COUNT): + break + + if attempt < MAX_RETRIES: + logger.warning( + "Attempt %d: PFC counter not updated for interface %s priority %d " + "(got %s), retrying send", attempt, intf, priority, + pfc_rx[intf]['Rx'][priority]) + duthost.sonic_pfc_counters(method="clear") + send_frames() + + else: + time.sleep(5) + pfc_rx = duthost.sonic_pfc_counters(method="get")['ansible_facts'] - pfc_rx = duthost.sonic_pfc_counters( - method="get")['ansible_facts'] if asic_type != 'vs': """check pfc Rx frame count on particular priority are increased""" assert pfc_rx[intf]['Rx'][priority] == str(PKT_COUNT), ( diff --git a/tests/common/helpers/pfc_gen_brcm_xgs.py b/tests/common/helpers/pfc_gen_brcm_xgs.py index 1a6730513fc..e05d569eda2 100755 --- a/tests/common/helpers/pfc_gen_brcm_xgs.py +++ b/tests/common/helpers/pfc_gen_brcm_xgs.py @@ -97,7 +97,7 @@ def _parseInterfaceMapFullSonic(self): intfTolPort = {} lPortToIntf = {} - if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")): + if self.switchChip.startswith(("Tomahawk6", "Tomahawk5", "Tomahawk4")): output = self._bcmltshellCmd('knet netif info') for info in output.split("Network interface Info:"): mo = re.search(r"Name: (?PEthernet\d+)[\s\S]{1,100}Port: (?P\d+)", info) @@ -143,7 +143,7 @@ def _endPfcStorm(self, intf): ''' mmuPort = self.intfToMmuPort[intf] port = self.intfToPort[intf] - if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")): + if self.switchChip.startswith(("Tomahawk6", "Tomahawk5", "Tomahawk4")): self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0") self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=0") else: @@ -175,7 +175,7 @@ def startPfcStorm(self, intf): if (1 << prio) & self.priority: self._cliCmd(f"en\nconf\n\nint {intf}\npriority-flow-control priority {prio} no-drop") - if self.switchChip.startswith(("Tomahawk5", "Tomahawk4")): + if self.switchChip.startswith(("Tomahawk6", "Tomahawk5", "Tomahawk4")): self._bcmltshellCmd(f"pt MMU_INTFO_XPORT_BKP_HW_UPDATE_DISr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP=1") self._bcmltshellCmd(f"pt MMU_INTFO_TO_XPORT_BKPr set BCMLT_PT_PORT={mmuPort} PAUSE_PFC_BKP={self.priority}") else: diff --git a/tests/common/helpers/pfc_storm.py b/tests/common/helpers/pfc_storm.py index 9b8757d050c..6234e76bb37 100644 --- a/tests/common/helpers/pfc_storm.py +++ b/tests/common/helpers/pfc_storm.py @@ -32,6 +32,7 @@ def get_chip_name_if_asic_pfc_storm_supported(fanout): "Arista DCS-7260CX3": "Tomahawk2", "Arista-7260CX3": "Tomahawk2", "Arista-7260QX3": "Tomahawk2", + "Nokia-IXR7220": "Tomahawk6", } for sku, chip in hwSkuInfo.items(): diff --git a/tests/common/helpers/pfcwd_helper.py b/tests/common/helpers/pfcwd_helper.py index 92a0371670b..631041c20dd 100644 --- a/tests/common/helpers/pfcwd_helper.py +++ b/tests/common/helpers/pfcwd_helper.py @@ -9,7 +9,9 @@ from tests.ptf_runner import ptf_runner from tests.common import constants +from tests.common import config_reload from tests.common.cisco_data import is_cisco_device +from tests.common.devices.eos import EosHost from tests.common.mellanox_data import is_mellanox_device # If the version of the Python interpreter is greater or equal to 3, set the unicode variable to the str class. @@ -431,6 +433,31 @@ def fetch_vendor_specific_diagnosis_re(duthost): return VENDOR_SPEC_ADDITIONAL_INFO_RE.get(duthost.facts["asic_type"], "") +def is_pfcwd_hw_recovery_enabled(duthost): + """ + Check if PFC watchdog is using hardware-based recovery mechanism. + + Hardware-based recovery uses ASIC-level PFC DLR which controls egress/TX + traffic by ignoring PFC XOFF. The per-queue TX OK/DROP cells surfaced by + 'show pfcwd stats' are sourced from the software-recovery code path and + stay at 0 on HW-recovery platforms even when silicon is dropping. + + Returns: + bool: True if RECOVERY_MECHANISM is "hardware", False otherwise. + """ + try: + cmd = 'sonic-db-cli STATE_DB HGET "PFC_WD_STATE_TABLE|PFC_WD" "RECOVERY_MECHANISM"' + result = duthost.shell(cmd, module_ignore_errors=True) + output = result.get('stdout', '').strip().strip('"').strip("'").lower() + is_hardware = (output == "hardware") + logger.info("PFC watchdog recovery mechanism: {} (hardware={})".format( + output or "not set", is_hardware)) + return is_hardware + except Exception as e: + logger.error("Exception while checking recovery mechanism: {}".format(str(e))) + return False + + @pytest.fixture(scope='class', autouse=False) def start_background_traffic( duthosts, @@ -658,7 +685,7 @@ def _parse_pfcwd_stats(dut): Returns: dict: {(port, queue): {'status': str, 'storm_detect_count': int, 'restored_count': int}} """ - pfcwd_stat_output = dut.show_and_parse('show pfcwd stat') + pfcwd_stat_output = dut.show_and_parse('show pfcwd stats') stats_dict = {} for item in pfcwd_stat_output: @@ -695,7 +722,7 @@ def _get_storm_test_ports(storm_hndle): def verify_all_ports_pfc_storm_in_expected_state(dut, storm_hndle, expected_state, selected_test_ports, baseline_counters=None, threshold_percentage=100, - stormed_ports_list=None): + stormed_ports_list=None, test_ports_info=None): """Verify if threshold percentage of ports reached expected PFC storm state.""" if dut.facts['asic_type'] == 'vs': return True @@ -717,6 +744,9 @@ def verify_all_ports_pfc_storm_in_expected_state(dut, storm_hndle, expected_stat # Verify each port ports_in_expected_state = 0 + # Track per-port result so the duplicate-neighbor grouping below can recompute + # both the numerator and denominator consistently. + port_results = {} for test_port, queue_idx in ports_to_check: port_stats = pfcwd_stats_dict.get((test_port, queue_idx)) @@ -743,6 +773,8 @@ def verify_all_ports_pfc_storm_in_expected_state(dut, storm_hndle, expected_stat if ("storm" not in current_status) and (current_detect_count == current_restored_count): is_in_expected_state = True + port_results[test_port] = port_results.get(test_port, False) or is_in_expected_state + if is_in_expected_state: ports_in_expected_state += 1 if expected_state == "storm" and stormed_ports_list is not None and test_port not in stormed_ports_list: @@ -755,6 +787,49 @@ def verify_all_ports_pfc_storm_in_expected_state(dut, storm_hndle, expected_stat logger.warning("No ports found to verify") return False + # On non-dualtor t0 topologies, setup_pfc_test assigns a single VLAN neighbor IP + # (self.vlan_nw) to every VLAN port, so at most one of those ports can actually + # receive PTF background traffic -- the DUT does one ND/ARP lookup and routes all + # traffic out a single port. Counting every such VLAN port individually against the + # storm threshold inflates the denominator and causes false failures. + # + # To keep the numerator and denominator consistent, group VLAN ports that share a + # test_neighbor_addr and treat each group as one "effective" port (success = any + # member reached the expected state). Non-VLAN ports (routed interfaces and + # PortChannel members, which also share a neighbor IP by design in parse_pc_list) + # are always counted individually. This adjustment only applies to the storm phase. + if expected_state == "storm" and test_ports_info: + vlan_ip_groups = {} + standalone_ports = [] + seen_ports = set() + for port, _queue_idx in ports_to_check: + if port in seen_ports: + continue + seen_ports.add(port) + info = test_ports_info.get(port, {}) or {} + ip = info.get('test_neighbor_addr') + if info.get('test_port_type') == 'vlan' and ip: + vlan_ip_groups.setdefault(ip, []).append(port) + else: + standalone_ports.append(port) + + # Only adjust when VLAN ports actually share an IP (the non-dualtor case). + if any(len(ports) > 1 for ports in vlan_ip_groups.values()): + effective_total = len(vlan_ip_groups) + len(standalone_ports) + effective_success = 0 + for ip, ports in vlan_ip_groups.items(): + if any(port_results.get(p, False) for p in ports): + effective_success += 1 + for port in standalone_ports: + if port_results.get(port, False): + effective_success += 1 + logger.info( + "Adjusting for duplicate VLAN neighbor IPs: ports_in_expected_state %d->%d, " + "total_ports %d->%d", + ports_in_expected_state, effective_success, total_ports, effective_total) + ports_in_expected_state = effective_success + total_ports = effective_total + success_percentage = (ports_in_expected_state / total_ports) * 100 logger.info(f"{ports_in_expected_state}/{total_ports} ports ({success_percentage:.1f}%) " f"in '{expected_state}' state (threshold: {threshold_percentage}%)") @@ -789,7 +864,7 @@ def parser_show_pfcwd_stat(dut, select_port, select_queue): admin@bjw-can-7060-1:~$ """ logger.info("port {} queue {}".format(select_port, select_queue)) - pfcwd_stat_output = dut.show_and_parse('show pfcwd stat') + pfcwd_stat_output = dut.show_and_parse('show pfcwd stats') pfcwd_stat = [] for item in pfcwd_stat_output: @@ -852,3 +927,140 @@ def pfcwd_show_status(duthost, output_string): logger.debug("execute cmd {} response: \n{}".format(cmd, cmd_response.get('stdout', None))) return + + +def send_tx_egress(traffic_inst, action, verify, async_mode=False, pkt_count=None): + """Send traffic from the Rx port toward the Tx (egress) port and optionally verify + that the expected PFC watchdog action (forward/drop) is observed on egress.""" + logger.info("Check for egress {} on Tx port {} (verify={})".format(action, traffic_inst.pfc_wd_test_port, verify)) + dst_port = "[" + str(traffic_inst.pfc_wd_test_port_id) + "]" + if action == "forward" and type(traffic_inst.pfc_wd_test_port_ids) == list: + dst_port = "".join(str(traffic_inst.pfc_wd_test_port_ids)).replace(',', '') + ptf_params = {'router_mac': traffic_inst.router_mac, + 'vlan_mac': traffic_inst.vlan_mac, + 'queue_index': traffic_inst.pfc_queue_index, + 'pkt_count': pkt_count or traffic_inst.pfc_wd_test_pkt_count, + 'port_src': traffic_inst.pfc_wd_rx_port_id[0], + 'port_dst': dst_port, + 'ip_dst': traffic_inst.pfc_wd_test_neighbor_addr, + 'port_type': traffic_inst.port_id_to_type_map[traffic_inst.pfc_wd_rx_port_id[0]], + 'wd_action': action if verify else "dontcare", + 'ip_version': traffic_inst.ip_version} + if traffic_inst.pfc_wd_rx_port_vlan_id is not None: + ptf_params['port_src_vlan_id'] = traffic_inst.pfc_wd_rx_port_vlan_id + if traffic_inst.pfc_wd_test_port_vlan_id is not None: + ptf_params['port_dst_vlan_id'] = traffic_inst.pfc_wd_test_port_vlan_id + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(traffic_inst.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file, is_python3=True, async_mode=async_mode) + + +def shutdown_lag_members(duthost, selected_port, tbinfo, nbrhosts, ports): + """Shut down all LAG members except the selected port so that PFC watchdog + testing runs over a single link while keeping the port-channel up + (min-links=1). + + Multi-asic-aware: uses minigraph_portchannels (frontend port names on both + single-asic and multi-asic) instead of config_facts['PORTCHANNEL_MEMBER'] + (which on multi-asic is keyed by asic-internal names like Ethernet1/1 + that don't match the frontend names in test_ports/vm_neighbors). All DUT + config edits go through namespace-aware CLI/sonic-db-cli; no on-disk + config_db.json edits, so restore can revert via config_reload. + """ + if ports[selected_port]['test_port_type'] != 'portchannel': + return None, None, None + + dst_mgfacts = duthost.get_extended_minigraph_facts(tbinfo) + portChannel = None + portChannelMembers = [] + for pc_name, pc_meta in dst_mgfacts['minigraph_portchannels'].items(): + if selected_port in pc_meta.get('members', []): + portChannel = pc_name + portChannelMembers = list(pc_meta['members']) + break + if portChannel is None: + return None, None, None + + vm_neighbors = dst_mgfacts['minigraph_neighbors'] + peer_device = vm_neighbors[portChannelMembers[0]]['name'] + peer_port = vm_neighbors[portChannelMembers[0]]['port'] + vm_host = nbrhosts[peer_device]['host'] + + neigh_port_channel = None + min_links = None + if isinstance(vm_host, EosHost): + neigh_port_channels = vm_host.eos_command( + commands=['show port-channel | json'])['stdout'][0]["portChannels"] + for po_name, po_config in neigh_port_channels.items(): + for member in po_config['activePorts']: + if member == peer_port: + neigh_port_channel = po_name + min_links = len(po_config['activePorts']) + break + vm_host.eos_config(lines=['port-channel min-links 1'], + parents=[f'int {neigh_port_channel}']) + + # Namespace-aware CLI option: '' on single-asic, '-n asicN' on multi-asic. + ns = duthost.get_port_asic_instance(selected_port).cli_ns_option + # Drop min-links to 1 so the LAG stays up while N-1 members are shut. + duthost.shell( + f"sonic-db-cli {ns} CONFIG_DB hset 'PORTCHANNEL|{portChannel}' min_links 1") + for port in portChannelMembers: + if port == selected_port: + continue + duthost.shell(f"sudo config interface {ns} shutdown {port}") + + return vm_host, neigh_port_channel, min_links + + +def restore_original_config(duthost, selected_port, vm_host, neigh_port_channel, min_links, ports): + """Revert LAG/min-links edits made by shutdown_lag_members. + + Since shutdown_lag_members modifies running CONFIG_DB only (no on-disk + edits), config_reload from the on-disk config_db is sufficient to bring + members back up and restore the original min_links. + """ + if ports[selected_port]['test_port_type'] != 'portchannel': + return + + if isinstance(vm_host, EosHost): + vm_host.eos_config(lines=[f'port-channel min-links {min_links}'], + parents=[f'int {neigh_port_channel}']) + + config_reload(duthost, config_source='config_db', safe_reload=True, + check_intf_up_ports=True, wait_for_bgp=True) + + +def _is_multi_member_lag(duthost, port, ports): + if ports[port]['test_port_type'] != 'portchannel': + return False + pc_members = duthost.config_facts( + host=duthost.hostname, source="persistent" + )['ansible_facts'].get('PORTCHANNEL_MEMBER', {}) + return any(port in members and len(members) > 1 for members in pc_members.values()) + + +@pytest.fixture(scope='module') +def manage_lag_config(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, nbrhosts, setup_pfc_test): + """LAG config resource manager for PFCwd tests. + + Setup: shuts down extra LAG members so only the selected port remains active. + Teardown: restores the original config_db; runs even on test failure. + Yields (vm_host, neigh_port_channel, min_links). Skips setup/teardown when + the selected port is not in a multi-member portchannel. + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + ports = setup_pfc_test['selected_test_ports'] + port = list(ports.keys())[0] + + if not _is_multi_member_lag(duthost, port, ports): + yield None, None, None + return + + vm_host, neigh_port_channel, min_links = shutdown_lag_members( + duthost, port, tbinfo, nbrhosts, ports) + try: + yield vm_host, neigh_port_channel, min_links + finally: + restore_original_config(duthost, port, vm_host, neigh_port_channel, min_links, ports) diff --git a/tests/common/helpers/platform_api/fan_discrete_speed_helper.py b/tests/common/helpers/platform_api/fan_discrete_speed_helper.py new file mode 100644 index 00000000000..23deddfa580 --- /dev/null +++ b/tests/common/helpers/platform_api/fan_discrete_speed_helper.py @@ -0,0 +1,90 @@ +""" +Shared helpers for chassis / fan-drawer fan tests on platforms with discrete PWM steps +from `chassis.fans.supported_speeds` in platform.json (via `duthost.facts`). +""" + +import logging +import random + +from tests.common.helpers.platform_api import fan, fan_drawer_fan + +logger = logging.getLogger(__name__) + +# Fan - for these SKUs; use `chassis.fans.supported_speeds` in platform.json +FAN_SPEED_DISCRETE_PLATFORMS = ( + "x86_64-nokia_ixr7220_h6_128-r0", + "x86_64-nokia_ixr7220_h6_64-r0", +) + + +def fan_speed_uses_platform_discrete_list(duthost): + return duthost.facts.get("platform") in FAN_SPEED_DISCRETE_PLATFORMS + + +def parse_supported_speeds_csv(raw, log_ctx): + """Parse comma-separated integer speeds from platform.json.""" + if raw is None: + return None + text = raw if isinstance(raw, str) else str(raw) + out = [] + for part in text.split(","): + part = part.strip() + if not part: + continue + try: + out.append(int(part)) + except ValueError: + logger.warning( + "Ignoring non-integer supported_speeds token %r (%s)", + part, + log_ctx, + ) + return out or None + + +def get_chassis_fans_supported_speeds(duthost): + """ + Read the common discrete fan table from `chassis.fans.supported_speeds` in platform.json (via duthost.facts). + """ + chassis = duthost.facts.get("chassis") or {} + fans = chassis.get("fans") + if not fans or not isinstance(fans, list): + return None + for entry in fans: + if not isinstance(entry, dict): + continue + raw = entry.get("supported_speeds") + if raw is not None: + return parse_supported_speeds_csv(raw, "chassis.fans supported_speeds") + return None + + +def pick_initial_discrete_target_speed(num_fans, chassis_speeds, is_controllable, get_smin, get_smax): + """ + Pick one discrete speed using the first controllable fan index for min/max bounds. + """ + if not chassis_speeds: + return None + for probe in range(num_fans): + if not is_controllable(probe): + continue + smin = get_smin(probe) + smax = get_smax(probe) + in_range = [s for s in chassis_speeds if smin <= s <= smax] + pick_from = in_range if in_range else chassis_speeds + return random.choice(pick_from) + return None + + +def fan_drawer_speed_within_tolerance(platform_api_conn, j, i): + """True when fan-drawer fan speed is within platform API under/over tolerance for target.""" + under = fan_drawer_fan.is_under_speed(platform_api_conn, j, i) + over = fan_drawer_fan.is_over_speed(platform_api_conn, j, i) + return not under and not over + + +def chassis_fan_speed_within_tolerance(platform_api_conn, i): + """True when chassis fan speed is within platform API under/over tolerance for target.""" + under = fan.is_under_speed(platform_api_conn, i) + over = fan.is_over_speed(platform_api_conn, i) + return not under and not over diff --git a/tests/common/macsec/__init__.py b/tests/common/macsec/__init__.py index 8dfc76a6394..e7cee9c7cba 100644 --- a/tests/common/macsec/__init__.py +++ b/tests/common/macsec/__init__.py @@ -18,10 +18,15 @@ from .macsec_config_helper import cleanup_macsec_configuration from .macsec_config_helper import is_macsec_configured from .macsec_config_helper import get_macsec_enable_status, get_macsec_profile -from .macsec_helper import load_all_macsec_info +from .macsec_config_helper import generate_macsec_profile +from .macsec_config_helper import setup_macsec_multi_profile_configuration +from .macsec_config_helper import cleanup_macsec_multi_profile_configuration +from .macsec_config_helper import enable_macsec_port +from .macsec_helper import load_all_macsec_info, getns_prefix # flake8: noqa: F401 from tests.common.plugins.sanity_check import sanity_check +from tests.common.utilities import wait_until logger = logging.getLogger(__name__) @@ -64,6 +69,32 @@ def downstream_neighbor(self,tbinfo, neighbor): def upstream_neighbor(self,tbinfo, neighbor): return NotImplementedError() + @pytest.fixture(scope="module") + def port_profiles(self, request, ctrl_links, macsec_profile): + """Per-port profile mapping + + Returns ``None`` in single-profile mode. + When ``--per_interface_macsec`` is set, generates a unique + ``MACSEC_PROFILE_`` for every controlled port using the same + cipher_suite, policy, send_sci, priority, and rekey_period as the base + ``--macsec_profile``, but with unique CAK/CKN per port. + """ + if not request.config.getoption("per_interface_macsec", default=False): + return None + if len(ctrl_links) < 2: + pytest.skip("Per-interface profile tests require at least 2 controlled links") + profiles = {} + for dut_port in ctrl_links: + profiles[dut_port] = generate_macsec_profile( + port_name=dut_port, + cipher_suite=macsec_profile["cipher_suite"], + priority=macsec_profile["priority"], + policy=macsec_profile["policy"], + send_sci=macsec_profile["send_sci"], + rekey_period=macsec_profile["rekey_period"], + ) + return profiles + @pytest.fixture(scope="module") def start_macsec_service(self, macsec_duthost, macsec_nbrhosts): def __start_macsec_service(): @@ -83,7 +114,7 @@ def macsec_feature(self, start_macsec_service, stop_macsec_service): stop_macsec_service() @pytest.fixture(scope="module") - def startup_macsec(self, request, macsec_duthost, ctrl_links, macsec_profile, tbinfo): + def startup_macsec(self, request, macsec_duthost, ctrl_links, macsec_profile, port_profiles, tbinfo): topo_name = tbinfo['topo']['name'] def __startup_macsec(): profile = macsec_profile @@ -94,42 +125,80 @@ def __startup_macsec(): # will drop it for macsec kernel module does not correctly handle it. pytest.skip( "macsec on dut vsonic, neighbor eos, send_sci false") - if 't2' not in topo_name: - cleanup_macsec_configuration(macsec_duthost, ctrl_links, profile['name']) - setup_macsec_configuration(macsec_duthost, ctrl_links, - profile['name'], profile['priority'], profile['cipher_suite'], - profile['primary_cak'], profile['primary_ckn'], profile['policy'], - profile['send_sci'], profile['rekey_period'], tbinfo) + + if port_profiles: + # Save original profile bindings so shutdown can restore them. + self._original_profile_per_port = {} + for dut_port in ctrl_links: + ns = getns_prefix(macsec_duthost, dut_port) + cmd = "sonic-db-cli {} CONFIG_DB HGET 'PORT|{}' 'macsec'".format( + ns, dut_port) + output = macsec_duthost.command(cmd)['stdout'].strip() + self._original_profile_per_port[dut_port] = output if output else None + + setup_macsec_multi_profile_configuration( + macsec_duthost, ctrl_links, port_profiles, tbinfo) + + logger.info( + "Setup per-interface MACsec configuration with profiles:\n{}".format( + {p: pp["name"] for p, pp in port_profiles.items()})) + else: + + if 't2' not in topo_name: + cleanup_macsec_configuration(macsec_duthost, ctrl_links, profile['name']) + setup_macsec_configuration(macsec_duthost, ctrl_links, + profile['name'], profile['priority'], profile['cipher_suite'], + profile['primary_cak'], profile['primary_ckn'], profile['policy'], + profile['send_sci'], profile['rekey_period'], tbinfo) logger.info( "Setup MACsec configuration with arguments:\n{}".format(locals())) + return __startup_macsec @pytest.fixture(scope="module") - def shutdown_macsec(self, macsec_duthost, ctrl_links, macsec_profile): + def shutdown_macsec(self, macsec_duthost, ctrl_links, macsec_profile, port_profiles, tbinfo): def __shutdown_macsec(): profile = macsec_profile - cleanup_macsec_configuration(macsec_duthost, ctrl_links, profile['name']) + if port_profiles: + cleanup_macsec_multi_profile_configuration( + macsec_duthost, ctrl_links, port_profiles) + # Restore original profile bindings. + orig = getattr(self, '_original_profile_per_port', {}) + for dut_port, nbr in list(ctrl_links.items()): + orig_name = orig.get(dut_port) + if orig_name: + enable_macsec_port(macsec_duthost, dut_port, orig_name) + enable_macsec_port(nbr["host"], nbr["port"], orig_name) + for dut_port, nbr in list(ctrl_links.items()): + # only check the port if it was actually put back to an old macsec configuration + if orig.get(dut_port): + wait_until(300, 3, 0, + lambda dp=dut_port, n=nbr: macsec_duthost.iface_macsec_ok(dp) and + n["host"].iface_macsec_ok(n["port"])) + else: + cleanup_macsec_configuration(macsec_duthost, ctrl_links, profile['name']) return __shutdown_macsec @pytest.fixture(scope="module") - def macsec_setup(self, startup_macsec, shutdown_macsec, macsec_feature, macsec_duthost, macsec_profile, ctrl_links): + def macsec_setup(self, startup_macsec, shutdown_macsec, macsec_feature, macsec_duthost, macsec_profile, port_profiles, ctrl_links): ''' setup macsec links ''' shutdown = False if get_macsec_enable_status(macsec_duthost) and get_macsec_profile(macsec_duthost): + macsec_preconfigured = is_macsec_configured(macsec_duthost, macsec_profile, ctrl_links) - if not macsec_preconfigured: + if not macsec_preconfigured or port_profiles is not None: shutdown = True startup_macsec() - else: + if macsec_preconfigured: logger.info(f"Macsec is already configured for {macsec_profile}, skipping setup") yield if shutdown: shutdown_macsec() @pytest.fixture(scope="module", autouse=True) - def load_macsec_info(self, request, macsec_setup, ctrl_links, macsec_duthost, tbinfo): + def load_macsec_info(self, request, macsec_setup, macsec_duthost, ctrl_links, macsec_profile, port_profiles, tbinfo): """Pre-load MACsec session info for all control links. If MACsec is enabled and configured for this DUT/profile, wait for @@ -144,6 +213,7 @@ def load_macsec_info(self, request, macsec_setup, ctrl_links, macsec_duthost, tb request.getfixturevalue('wait_mka_establish') except pytest.FixtureLookupError: pass + load_all_macsec_info(macsec_duthost, ctrl_links, tbinfo) @pytest.fixture(scope="module") diff --git a/tests/common/macsec/macsec_config_helper.py b/tests/common/macsec/macsec_config_helper.py index 58eae0e3865..a980410d0e2 100644 --- a/tests/common/macsec/macsec_config_helper.py +++ b/tests/common/macsec/macsec_config_helper.py @@ -1,5 +1,8 @@ import logging +import secrets import time +from passlib.hash import cisco_type7 + from tests.common.macsec.macsec_helper import get_mka_session, getns_prefix, wait_all_complete, \ submit_async_task from tests.common.macsec.macsec_platform_helper import global_cmd, find_portchannel_from_member, get_portchannel @@ -17,7 +20,10 @@ 'disable_macsec_port', 'get_macsec_enable_status', 'get_macsec_profile', - 'wait_for_macsec_cleanup' + 'wait_for_macsec_cleanup', + 'generate_macsec_profile', + 'setup_macsec_multi_profile_configuration', + 'cleanup_macsec_multi_profile_configuration', ] logger = logging.getLogger(__name__) @@ -177,6 +183,12 @@ def disable_macsec_port(host, port): host.command("sudo config portchannel {} member add {} {}".format(getns_prefix(host, port), pc["name"], port)) +def replace_macsec_port(host, port, profile_name): + disable_macsec_port(host, port) + time.sleep(10) + enable_macsec_port(host, port, profile_name) + + def enable_macsec_feature(duthost, macsec_nbrhosts): nbrhosts = macsec_nbrhosts num_asics = duthost.num_asics() @@ -286,6 +298,169 @@ def setup_macsec_configuration(duthost, ctrl_links, profile_name, default_priori logger.info("Setup macsec configuration finished") +def generate_macsec_profile(port_name, cipher_suite="GCM-AES-128", priority=64, + policy="security", send_sci="true", rekey_period=0): + """Generate a MACsec profile with random CAK/CKN for a specific port. + + The profile is named ``MACSEC_PROFILE_`` and the pre-shared keys + are generated using ``secrets.token_hex`` so that every port receives a + unique key pair. + + Args: + port_name: Interface name (e.g. "Ethernet0"). Used in the profile name. + cipher_suite: Cipher suite string. Determines key lengths. + priority: MKA key-server priority (0-255). + policy: "security" (encrypt) or "integrity" (auth only). + send_sci: "true" or "false". + rekey_period: Seconds between rekeying (0 = disabled). + + Returns: + dict: A profile dict compatible with set_macsec_profile(). + """ + + # CAK length: AES-128 variants use 32 bytes (66 hex chars), + # AES-256 variants use 64 bytes (130 hex chars). + # CKN length: AES-128 variants use 16 bytes (32 hex chars), + # AES-256 variants use 32 bytes (64 hex chars). + if "128" in cipher_suite: + cak = secrets.token_hex(16) + # token_hex produces a string of n*2 chars (as each hex num is 2 chars) + # For CKN, this is interpreted as the literal password, so when passed to + # the type7 encoder each hex char is treated as its own byte. + # This is why the number passed to token hex is half the expected number of bytes, + # because the length of the string generated is double + ckn = secrets.token_hex(16) + else: + cak = secrets.token_hex(32) + ckn = secrets.token_hex(32) + + # CAK is expected to be in "type7" encoding format + # This adds the extra byte to the cak / ckn length + cak = cisco_type7.hash(cak) + + profile_name = "MACSEC_PROFILE_{}".format(port_name) + return { + "name": profile_name, + "priority": priority, + "cipher_suite": cipher_suite, + "primary_cak": cak, + "primary_ckn": ckn, + "policy": policy, + "send_sci": send_sci, + "rekey_period": rekey_period, + } + + +def setup_macsec_multi_profile_configuration(duthost, ctrl_links, port_profiles, tbinfo): + """Set up MACsec with a different profile per port. + + Each port in *ctrl_links* is configured with its own profile from + *port_profiles*. The DUT uses ``default_priority`` from the profile while + neighbors alternate between ``priority - 1`` and ``priority + 1`` so the + DUT is elected MKA key-server on most links. + + Args: + duthost: DUT host object. + ctrl_links: dict ``{dut_port: {name, host, port, ...}}``. + port_profiles: dict ``{dut_port: profile_dict}`` where each + ``profile_dict`` has keys matching ``generate_macsec_profile`` + output. + tbinfo: Testbed info dict. + """ + logger.info("Multi-profile setup step 1: set per-port macsec profiles") + + for dut_port, profile in port_profiles.items(): + set_macsec_profile( + duthost, profile["name"], profile["priority"], + profile["cipher_suite"], profile["primary_cak"], + profile["primary_ckn"], profile["policy"], + profile["send_sci"], profile["rekey_period"]) + i = 0 + for dut_port, nbr in ctrl_links.items(): + profile = port_profiles[dut_port] + + if i % 2 == 0: + nbr_priority = profile["priority"] - 1 + else: + nbr_priority = profile["priority"] + 1 + set_macsec_profile( + nbr["host"], profile["name"], nbr_priority, + profile["cipher_suite"], profile["primary_cak"], + profile["primary_ckn"], profile["policy"], + profile["send_sci"], profile["rekey_period"]) + i += 1 + time.sleep(3) + + logger.info("Multi-profile setup step 2: enable per-port macsec") + + for dut_port, nbr in list(ctrl_links.items()): + profile = port_profiles[dut_port] + time.sleep(3) + enable_macsec_port(duthost, dut_port, profile["name"]) + enable_macsec_port(nbr["host"], nbr["port"], profile["name"]) + + logger.info("Multi-profile setup step 3: wait for macsec ready on each port") + + for dut_port, nbr in list(ctrl_links.items()): + assert wait_until(300, 3, 0, + lambda dp=dut_port, n=nbr: duthost.iface_macsec_ok(dp) and + n["host"].iface_macsec_ok(n["port"])) + + # Hold time for protocol recovery after link flaps. + time.sleep(60) + logger.info("Multi-profile setup finished") + + +def cleanup_macsec_multi_profile_configuration(duthost, ctrl_links, port_profiles): + """Clean up per-port MACsec profiles. + + Disables MACsec on every controlled port, then deletes each unique profile + from CONFIG_DB on both the DUT and neighbor devices. + + Args: + duthost: DUT host object. + ctrl_links: dict ``{dut_port: {name, host, port, ...}}``. + port_profiles: dict ``{dut_port: profile_dict}``. + """ + devices = set() + if duthost.facts["asic_type"] == "vs": + devices.add(duthost) + + logger.info("Multi-profile cleanup step 1: disable macsec on all ports") + for dut_port, nbr in list(ctrl_links.items()): + time.sleep(3) + disable_macsec_port(duthost, dut_port) + disable_macsec_port(nbr["host"], nbr["port"]) + devices.add(nbr["host"]) + + logger.info("Multi-profile cleanup step 2: delete per-port profiles") + deleted_profiles = set() + for dut_port, nbr in list(ctrl_links.items()): + profile_name = port_profiles[dut_port]["name"] + if profile_name not in deleted_profiles: + delete_macsec_profile(duthost, profile_name) + deleted_profiles.add(profile_name) + + for d in devices: + for profile_name in deleted_profiles: + delete_macsec_profile(d, profile_name) + + logger.info("Multi-profile cleanup step 3: wait for automatic cleanup") + + interfaces = list(ctrl_links.keys()) + wait_for_macsec_cleanup(duthost, interfaces) + + for dut_port, nbr in list(ctrl_links.items()): + wait_for_macsec_cleanup(nbr["host"], [nbr["port"]]) + + logger.info("Multi-profile cleanup finished") + + for d in devices: + if isinstance(d, EosHost): + continue + assert wait_until(30, 1, 0, lambda d=d: not get_mka_session(d)) + + def wait_for_macsec_cleanup(host, interfaces, timeout=90): """Wait for MACsec daemon to automatically clean up all MACsec entries. diff --git a/tests/common/platform/device_utils.py b/tests/common/platform/device_utils.py index 5f844ddb1b7..a8d5e228bf9 100755 --- a/tests/common/platform/device_utils.py +++ b/tests/common/platform/device_utils.py @@ -380,8 +380,17 @@ def check_neighbors(duthost, tbinfo): mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + # Check if this topo includes confed peer + confed_peer_topo = False + for v in bgp_facts['bgp_neighbors'].values(): + if v.get('confed_peer', False): + confed_peer_topo = True + break + for value in list(bgp_facts['bgp_neighbors'].values()): # Verify locat ASNs in bgp sessions + if confed_peer_topo and (not value.get("confed_peer", False)): + continue if (value['local AS'] != mg_facts['minigraph_bgp_asn']): raise RebootHealthError("Local ASNs not found in BGP session.\ Minigraph: {}. Found {}".format(value['local AS'], mg_facts['minigraph_bgp_asn'])) diff --git a/tests/common/plugins/conditional_mark/__init__.py b/tests/common/plugins/conditional_mark/__init__.py index abf9d7383f8..7a0f5b8992e 100644 --- a/tests/common/plugins/conditional_mark/__init__.py +++ b/tests/common/plugins/conditional_mark/__init__.py @@ -30,8 +30,9 @@ 't1-backend', 't1-isolated-d128', 't1-isolated-d32', 't2', 't2_2lc_36p-masic', 't2_2lc_min_ports-masic', 'lt2-p32o64', 'lt2-o128', 'ft2-64', 'ft2-16', 't2_one_hwsku_min', 't2_one_hwsku_max', - 't2-single-node-min', 't2_single_node_max', 't2_single_node_max_64p', 't2-single-node-max-64p', - 'topo_t2_single_node_max_64p_v2', 'urh_min', 'lrh_min'] + 't2-single-node-min', 't2_single_node_min', 't2_single_node_max', + 't2_single_node_max_64p', 't2-single-node-max-64p', + 't2_single_node_max_64p_v2', 'urh_min', 'lrh_min'] } diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index aa2bc8d2368..5477e8a0e95 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -17,17 +17,26 @@ 't0-isolated-d16u16s1', 't0-isolated-v6-d16u16s1', 't0-isolated-d16u16s2', 't0-isolated-v6-d16u16s2', 't0-isolated-d256u256s2', 't0-isolated-v6-d256u256s2', + 't0-isolated-d32u32s2-mix', 't0-isolated-d32u32s2', 't0-isolated-v6-d32u32s2', + 't0-isolated-d96u32s2', 't0-isolated-v6-d96u32s2', + 't1-isolated-v6-d128', 't1-isolated-d224u8', 't1-isolated-v6-d224u8', + 't1-isolated-d254u2', 't1-isolated-d254u2s1', + 't1-isolated-d254u2s2', 't1-isolated-d28u1', 't1-isolated-v6-d28u1', 't1-isolated-d28u4', + 't1-isolated-d32u1s2', 't1-isolated-d448u15-lag', 't1-isolated-v6-d448u15-lag', 't1-isolated-d448u16', 't1-isolated-v6-d448u16', + 't1-isolated-d508u1s2', 't1-isolated-d510u2', + 't1-isolated-d510u2s2', 't1-isolated-d56u1-lag', 't1-isolated-v6-d56u1-lag', - 't1-isolated-d56u2', 't1-isolated-v6-d56u2' ] + 't1-isolated-d56u2', 't1-isolated-v6-d56u2', + 't2-isolated-d128s2' ] - &noVxlanTopos | topo_name in [ - 't0-isolated-d32u32s2', 't0-isolated-d256u256s2', + 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix', 't0-isolated-d256u256s2', 't0-isolated-d96u32s2', 't0-isolated-v6-d32u32s2', 't0-isolated-v6-d256u256s2', 't0-isolated-v6-d96u32s2', 't1-isolated-d56u2', 't1-isolated-d56u1-lag', @@ -247,6 +256,7 @@ arp/test_wr_arp.py: - "'standalone' in topo_name" - "is_mgmt_ipv6_only==True" # Does not support ipv6 mgmt ip on dut, specially the ferret server - "'isolated' in topo_name" + - *lossyTopos - "topo_type not in ['t0']" - "'f2' in topo_name" @@ -265,9 +275,11 @@ autorestart/test_container_autorestart.py::test_containers_autorestart.*teamd.*: ####################################### bfd: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos bfd/test_bfd.py: skip: @@ -645,7 +657,9 @@ bgp/test_bgpmon.py::test_bgpmon: bgp/test_bgpmon_v6.py::test_bgpmon_no_ipv6_resolve_via_default: skip: - reason: "Not applicable for passive bgpmon_v6" + reason: "Not applicable for passive bgpmon_v6 and UT2 does not use bgpmon_v6" + conditions: + - "platform in ['x86_64-arista_7280dr3am_36']" bgp/test_startup_tsa_tsb_service.py::test_tsa_tsb_service_with_supervisor_abnormal_reboot: skip: @@ -739,9 +753,11 @@ clock/test_clock.py::test_config_clock_timezone: ####################################### configlet: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos configlet/test_add_rack.py: skip: @@ -761,6 +777,7 @@ console/: console/test_console_availability.py: skip: reason: "This test is applicable to virtual setup only." + conditions_logical_operator: OR conditions: - "asic_type in ['vpp'] and platform.startswith('arm64-c8220tg_48a_o')" - "platform in ['arm64-nokia_ixs7215_c1xa-r0']" @@ -781,7 +798,7 @@ copp/test_copp.py: skip: reason: "Topology not supported by COPP tests" conditions: - - "(topo_name not in ['dualtor-aa', 'dualtor-aa-64-breakout', 'ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" + - "(topo_name not in ['dualtor-aa', 'dualtor-aa-64-breakout', 'ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" xfail: reason: "xfail for IPv6-only topologies" conditions: @@ -793,7 +810,7 @@ copp/test_copp.py::TestCOPP::test_add_new_trap: conditions_logical_operator: or conditions: - "is_multi_asic==True" - - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" + - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" copp/test_copp.py::TestCOPP::test_remove_trap: skip: @@ -801,7 +818,7 @@ copp/test_copp.py::TestCOPP::test_remove_trap: conditions_logical_operator: or conditions: - "is_multi_asic==True" - - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" + - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" copp/test_copp.py::TestCOPP::test_trap_config_save_after_reboot: skip: @@ -811,7 +828,7 @@ copp/test_copp.py::TestCOPP::test_trap_config_save_after_reboot: - "is_multi_asic==True" - "build_version.split('.')[0].isdigit() and int(build_version.split('.')[0]) == 20220531 and int(build_version.split('.')[1]) > 27 and hwsku in ['Arista-7050-QX-32S', 'Arista-7050QX32S-Q32', 'Arista-7050-QX32', 'Arista-7050QX-32S-S4Q31', 'Arista-7060CX-32S-D48C8', 'Arista-7060CX-32S-C32', 'Arista-7060CX-32S-Q32', 'Arista-7060CX-32S-C32-T1']" - "build_version.split('.')[0].isdigit() and int(build_version.split('.')[0]) > 20220531 and hwsku in ['Arista-7050-QX-32S', 'Arista-7050QX32S-Q32', 'Arista-7050-QX32', 'Arista-7050QX-32S-S4Q31', 'Arista-7060CX-32S-D48C8', 'Arista-7060CX-32S-C32', 'Arista-7060CX-32S-Q32', 'Arista-7060CX-32S-C32-T1']" - - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" + - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't1', 't1-lag', 't1-28-lag', 't1-48-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx', 'm1-48', 'm1-44', 'm1-108', 'm1-128', 't0-isolated-d16u16s1', 't1-isolated-d28u1', 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix', 't1-isolated-d56u2'] and 't2' not in topo_type and topo_type not in ['lrh', 'urh'])" copp/test_copp.py::TestCOPP::test_trap_neighbor_miss: skip: @@ -819,7 +836,7 @@ copp/test_copp.py::TestCOPP::test_trap_neighbor_miss: conditions_logical_operator: or conditions: - "(asic_type in ['broadcom'] and release in ['202411'])" - - "(topo_name not in ['t0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't0-isolated-d16u16s1', 't0-isolated-d32u32s2'])" + - "(topo_name not in ['t0', 't0-64', 't0-52', 't0-116', 't0-118', 't0-88-o8c80', 't0-isolated-d16u16s1', 't0-isolated-d32u32s2', 't0-isolated-d32u32s2-mix'])" ####################################### ##### crm ##### @@ -854,6 +871,12 @@ crm/test_crm_available.py: ####################################### ##### dash ##### ####################################### +dash: + skip: + reason: "Dash tests are not supported on lossy topologies." + conditions: + - *lossyTopos + dash/crm/test_dash_crm.py: skip: reason: "Currently dash tests are not supported on KVM" @@ -964,7 +987,7 @@ decap/: decap/test_decap.py: skip: - reason: 'Skip on t1-isolated-d32/128 topos. Skip on Arista-720DT (TD3-X2 chip does not honor DSCP uniform decap; concession in aristanetworks/sonic-qual.msft#1176).' + reason: 'Skip on t1-isolated-d32/128 topos. Skip on Arista-720DT (TD3-X2 SAI does not support DSCP uniform decap).' conditions_logical_operator: or conditions: - "topo_name in ['t1-isolated-d128', 't1-isolated-d32']" @@ -1007,11 +1030,12 @@ decap/test_subnet_decap.py::test_vlan_subnet_decap: ####################################### dhcp_relay: skip: - reason: "Not applicable to isolated topologies." + reason: "Not applicable to isolated topologies, BMC, or lossy topologies." conditions_logical_operator: or conditions: - "'isolated' in topo_name" - "'bmc' in topo_type" + - *lossyTopos dhcp_relay/test_dhcp_counter_stress.py::test_dhcpmon_relay_counters_stress: xfail: @@ -1226,9 +1250,11 @@ dhcp_relay/test_dhcpv6_relay.py::test_interface_binding: ####################################### dhcp_server: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos ####################################### ##### disk ##### @@ -1302,9 +1328,11 @@ drop_packets/test_drop_counters.py::test_src_ip_is_multicast_addr: ####################################### dualtor: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos dualtor/test_bgp_block_loopback1.py: skip: @@ -1427,9 +1455,11 @@ dualtor/test_tunnel_memory_leak.py::test_tunnel_memory_leak: dualtor_io: skip: - reason: "Testcase could only be executed on dualtor testbed." + reason: "Testcase could only be executed on dualtor testbed and is not supported on lossy topologies." + conditions_logical_operator: or conditions: - "'dualtor' not in topo_name" + - *lossyTopos dualtor_io/test_grpc_server_failure.py: skip: @@ -1511,9 +1541,11 @@ dualtor_io/test_tor_failure.py: dualtor_mgmt: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos dualtor_mgmt/test_dualtor_bgp_update_delay.py: xfail: @@ -1926,11 +1958,11 @@ everflow/test_everflow_per_interface.py::test_everflow_per_interface[ipv6-m0_vla everflow/test_everflow_testbed.py::EverflowIPv4Tests::test_everflow_dscp_with_policer: skip: - reason: "Test not supported on Mellanox platforms or Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b) as policer is not supported. CS00012412189" + reason: "Test not supported on Mellanox platforms, Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b), or Nokia IXR7220-H6-128 as policer is not supported. CS00012412189" conditions_logical_operator: or conditions: - "asic_type in ['mellanox']" - - "platform in ['x86_64-arista_7060x6_64pe_b']" + - "platform in ['x86_64-arista_7060x6_64pe_b', 'x86_64-nokia_ixr7220_h6_128-r0']" everflow/test_everflow_testbed.py::TestEverflowV4EgressAclEgressMirror: skip: @@ -1979,12 +2011,12 @@ everflow/test_everflow_testbed.py::TestEverflowV4EgressAclEgressMirror::test_eve everflow/test_everflow_testbed.py::TestEverflowV4EgressAclEgressMirror::test_everflow_dscp_with_policer: skip: - reason: "Skipping test since mirror with policer is not supported on Cisco 8000 platforms, Broadcom DNX platforms, and Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b). CS00012412189" + reason: "Skipping test since mirror with policer is not supported on Cisco 8000 platforms, Broadcom DNX platforms, Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b), and Nokia IXR7220-H6-128. CS00012412189" conditions_logical_operator: "OR" conditions: - "asic_subtype in ['broadcom-dnx']" - "asic_type in ['cisco-8000']" - - "platform in ['x86_64-arista_7060x6_64pe_b']" + - "platform in ['x86_64-arista_7060x6_64pe_b', 'x86_64-nokia_ixr7220_h6_128-r0']" everflow/test_everflow_testbed.py::TestEverflowV4EgressAclEgressMirror::test_everflow_dscp_with_policer[erspan_ipv4-cli-downstream-default]: skip: @@ -2170,11 +2202,11 @@ everflow/test_everflow_testbed.py::TestEverflowV4IngressAclIngressMirror::test_e everflow/test_everflow_testbed.py::TestEverflowV4IngressAclIngressMirror::test_everflow_dscp_with_policer: skip: - reason: "Skipping test since mirror with policer is not supported on Cisco 8122 and 8223 platforms, Broadcom DNX platforms, and Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b). CS00012412189" + reason: "Skipping test since mirror with policer is not supported on Cisco 8122 and 8223 platforms, Broadcom DNX platforms, Broadcom Th5 platform (x86_64-arista_7060x6_64pe_b), and Nokia IXR7220-H6-128. CS00012412189" conditions_logical_operator: "OR" conditions: - "asic_subtype in ['broadcom-dnx']" - - "platform in ['x86_64-8122_64eh_o-r0', 'x86_64-8122_64ehf_o-r0', 'x86_64-arista_7060x6_64pe_b', 'x86_64-8223_64e_mo-r0','x86_64-8223_64ef_mo-r0']" + - "platform in ['x86_64-8122_64eh_o-r0', 'x86_64-8122_64ehf_o-r0', 'x86_64-arista_7060x6_64pe_b', 'x86_64-8223_64e_mo-r0','x86_64-8223_64ef_mo-r0', 'x86_64-nokia_ixr7220_h6_128-r0']" everflow/test_everflow_testbed.py::TestEverflowV4IngressAclIngressMirror::test_everflow_dscp_with_policer[erspan_ipv4-cli-downstream-default]: skip: @@ -2385,11 +2417,12 @@ fib/test_fib.py::test_ipinip_hash_negative[ipv4: fib/test_fib.py::test_nvgre_hash: skip: - reason: 'Nvgre hash test is not fully supported on VS and Broadcom platform; Not supported on M*. Skip on t1-isolated-d32/128 topos' + reason: 'Nvgre hash test is not fully supported on VS and Broadcom platform; Not supported on M* and cisco-8122, cisco-8223 platforms. Skip on t1-isolated-d32/128 topos' conditions_logical_operator: or conditions: - "asic_type in ['vs', 'broadcom'] or topo_type in ['m0', 'mx', 'm1']" - "topo_name in ['t1-isolated-d128', 't1-isolated-d32']" + - "platform in ['x86_64-8122_64eh_o-r0', 'x86_64-8122_64ehf_o-r0', 'x86_64-8223_64e_mo-r0', 'x86_64-8223_64ef_mo-r0']" xfail: reason: 'Nvgre hash test is not fully supported on SPC1 platform due to known limitation' conditions: @@ -2543,6 +2576,10 @@ generic_config_updater/test_bgp_prefix.py::test_bgp_prefix_tc1_suite: - "platform in ['x86_64-8122_64eh_o-r0', 'x86_64-8122_64ehf_o-r0']" - "asic_type in ['vs'] and https://github.com/sonic-net/sonic-mgmt/issues/18445" - "'isolated' in topo_name" + xfail: + reason: "BGP allowed-prefix GCU validation fails across platforms, tracked by https://github.com/sonic-net/sonic-mgmt/issues/25549" + conditions: + - "https://github.com/sonic-net/sonic-mgmt/issues/25549" generic_config_updater/test_bgp_speaker.py::test_bgp_speaker_tc1_test_config: xfail: @@ -2570,11 +2607,14 @@ generic_config_updater/test_dhcp_relay.py: generic_config_updater/test_dynamic_acl.py: skip: reason: "Device SKUs do not support the custom ACL_TABLE_TYPE that we use in this test / Known log error unrelated to test - on m0-2vlan testbed causes consistent failures." + on m0-2vlan testbed causes consistent failures. / Cisco 81XX devices are being skipped in production for 202511 and the patch from Cisco to fix their ACL key + has not been applied on 202511. 202501 it has been and tests will pass, and we expect this to be in future releases + as well." conditions_logical_operator: "OR" conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - "topo_name in ['m0-2vlan']" + - "release in ['202511'] and platform in ['x86_64-8101_32fh_o-r0', 'x86_64-8102_64h_o-r0']" generic_config_updater/test_dynamic_acl.py::test_gcu_acl_arp_rule_creation[IPV4: skip: @@ -2766,7 +2806,7 @@ generic_config_updater/test_pfcwd_interval.py: conditions_logical_operator: or conditions: - "asic_type not in ['mellanox', 'cisco-8000']" - - hwsku in ['Mellanox-SN5600-C224O8', 'Mellanox-SN5600-C256S1', 'Mellanox-SN5640-C448O16', 'Mellanox-SN5640-C512S2', + - hwsku in ['Mellanox-SN5600-C224O8', 'Mellanox-SN5600-C256S1', 'Mellanox-SN5640-C448O16', 'Mellanox-SN5640-C512S2', 'Mellanox-SN5640-C508O1X2', 'Arista-7060X6-64PE-C256S2', 'Arista-7060X6-64PE-C224O8', 'Arista-7060X6-64PE-B-C512S2', 'Arista-7060X6-64PE-B-C448O16'] - "'bmc' in topo_type" @@ -3390,9 +3430,11 @@ high_frequency_telemetry/test_high_frequency_telemetry.py::test_hft_full_counter ####################################### http: skip: - reason: "It is skipped for '202412' for now" + reason: "It is skipped for '202412' or lossy topologies for now" + conditions_logical_operator: or conditions: - "release in ['202412']" + - *lossyTopos ####################################### ##### iface_loopback_action ##### @@ -3619,10 +3661,11 @@ ipfwd/test_nhop_group.py::test_nhop_group_member_order_capability: ####################################### ixia: skip: - reason: "Ixia test only support on physical ixia testbed and it is not tested for now" + reason: "Ixia test only support on physical ixia testbed and it is not tested on lossy topologies for now" conditions_logical_operator: or conditions: - "asic_type in ['vs']" + - *lossyTopos - "True" ####################################### @@ -3721,9 +3764,11 @@ log_fidelity/: ####################################### macsec: skip: - reason: "Skip running on dualtor testbed" + reason: "Skip running on dualtor and lossy topologies" + conditions_logical_operator: or conditions: - "'dualtor' in topo_name" + - *lossyTopos macsec/test_dataplane.py::TestDataPlane::test_server_to_neighbor: skip: @@ -3748,9 +3793,11 @@ macsec/test_interop_protocol.py::TestInteropProtocol::test_snmp: ####################################### mclag: skip: - reason: "Skip running on dualtor testbed" + reason: "Skip running on dualtor and lossy topologies" + conditions_logical_operator: or conditions: - - "'dualtor' in topo_name" + - "'dualtor' in topo_name" + - *lossyTopos mclag/test_mclag_l3.py: skip: @@ -3774,9 +3821,11 @@ memory_checker/test_memory_checker.py: ####################################### mpls: skip: - reason: "MPLS feature is not enabled with image version, skipped" + reason: "MPLS feature is not enabled with image version, skipped on lossy topologies" + conditions_logical_operator: or conditions: - "'mpls' not in feature_status" + - *lossyTopos ####################################### ##### mvrf ##### @@ -3816,9 +3865,11 @@ mvrf/test_mgmtvrf.py::TestReboot::test_warmboot: ####################################### mx: skip: - reason: "Test is only applicable to m0, mx, and m1 topologies" + reason: "Test is only applicable to m0, mx, and m1 topologies, not lossy topologies" + conditions_logical_operator: or conditions: - "topo_type not in ['m0', 'mx', 'm1']" + - *lossyTopos ####################################### ##### nat ##### @@ -3836,11 +3887,12 @@ nat: ####################################### ospf: skip: - reason: "Neighbor type must be sonic, skip in PR testing and it is skipped for '202412' for now" + reason: "Neighbor type must be sonic, skip in PR testing and it is skipped for '202412' and lossy topologies for now" conditions_logical_operator: or conditions: - "asic_type in ['vs']" - "release in ['202412']" + - *lossyTopos ####################################### ##### override_config_table ##### @@ -3980,8 +4032,10 @@ pfc/test_unknown_mac.py: ####################################### pfc_asym: skip: - reason: "It is not tested for now" + reason: "It is not tested for now, including on lossy topologies" + conditions_logical_operator: or conditions: + - *lossyTopos - "True" pfc_asym/test_pfc_asym.py: @@ -4122,6 +4176,24 @@ platform_tests/api/test_sfp.py::TestSfpApi::test_get_transceiver_info: conditions: - "https://github.com/sonic-net/sonic-buildimage/issues/23426 and ('sn4700' in platform or 'sn4280' in platform)" +platform_tests/test_advanced_reboot.py: + skip: + reason: "Advanced reboot tests are not supported on lossy topologies." + conditions: + - *lossyTopos + +platform_tests/test_cont_warm_reboot.py: + skip: + reason: "Continuous warm reboot tests are not supported on lossy topologies." + conditions: + - *lossyTopos + +platform_tests/test_intf_fec.py::test_verify_fec_stats_counters: + skip: + reason: "Broadcom TH does not support FEC_SYMBOL_ERR at 50G" + conditions: + - "'Arista-7060CX-32S-D48C8'in hwsku" + platform_tests/test_reboot.py: skip: reason: "Found regression issues on multi-asic topology, temperarily skip this test case until the issues are addressed." @@ -4215,11 +4287,12 @@ qos: qos/test_buffer.py: skip: - reason: "These tests don't apply to cisco 8000 platforms or T2 or M* since they support only traditional model and it is skipped for '202412' for now" + reason: "These tests don't apply to cisco 8000 platforms, T2, M*, lossy topologies, or '202412' for now" conditions_logical_operator: or conditions: - "asic_type in ['cisco-8000'] or topo_type in ['t2', 'lrh', 'urh']" - "topo_type in ['m0', 'mx', 'm1']" + - *lossyTopos - "release in ['202412']" qos/test_buffer.py::test_buffer_model_test: @@ -4254,11 +4327,12 @@ qos/test_oq_watchdog.py: qos/test_pfc_counters.py: skip: - reason: "Not supported on SKU and It is skipped for '202412' for now" + reason: "Not supported on SKU, lossy topologies, and it is skipped for '202412' for now" conditions_logical_operator: or conditions: - "hwsku in ['Mellanox-SN5600-C256S1', 'Mellanox-SN5600-C224O8', 'Mellanox-SN5640-C512S2', 'Mellanox-SN5640-C448O16']" + - *lossyTopos - "release in ['202412']" qos/test_pfc_counters.py::test_pfc_unpause: @@ -4271,10 +4345,11 @@ qos/test_pfc_counters.py::test_pfc_unpause: qos/test_pfc_pause.py: skip: - reason: "This test is not run on this asic type or version or topology currently and also skip for 202412 for now" + reason: "This test is not run on this asic type, version, topology, or lossy topologies currently and also skip for 202412 for now" conditions_logical_operator: or conditions: - "topo_type not in ['t0']" + - *lossyTopos - "release in ['202412']" qos/test_pfc_pause.py::test_pfc_pause_lossless: @@ -4320,11 +4395,12 @@ qos/test_qos_dscp_mapping.py::TestQoSSaiDSCPQueueMapping_IPIP_Base::test_dscp_to qos/test_qos_masic.py: skip: - reason: "QoS tests for multi-ASIC only. Supported topos: t1-lag, t1-64-lag, t1-56-lag, t1-backend. / M* topo does not support qos. / KVM do not support swap syncd." + reason: "QoS tests for multi-ASIC only. Supported topos: t1-lag, t1-64-lag, t1-56-lag, t1-backend. / M* and lossy topo does not support qos. / KVM do not support swap syncd." conditions_logical_operator: or conditions: - "is_multi_asic==False or topo_name not in ['t1-lag', 't1-64-lag', 't1-56-lag', 't1-backend']" - "topo_type in ['m0', 'mx', 'm1']" + - *lossyTopos - "asic_type in ['vs']" qos/test_qos_probe.py: @@ -4351,6 +4427,12 @@ qos/test_qos_probe.py: # Mellanox SPC3 (Spectrum 3) - "hwsku in ['Mellanox-SN4600C-C64'] and https://github.com/sonic-net/sonic-mgmt/issues/22608" +qos/test_qos_probe.py::TestQosProbe::testQosHeadroomPoolProbe: + xfail: + reason: "HeadroomPool probe fails on Mellanox SPC1 and SPC3 starting at iteration #2: all packets sent to pg=4 (immediately after a successful pg=3 probe on the same src port) are ingress-dropped, so the probe algorithm cannot find a PFC threshold for pg=4. Tracked by #24558 (SPC1+SPC3) and #24215 (SPC1)." + conditions: + - "asic_gen in ['spc1', 'spc3'] and https://github.com/sonic-net/sonic-mgmt/issues/24558" + qos/test_qos_sai.py: skip: reason: "qos_sai tests not supported on t1 topo / M* topo does not support qos and It is skipped for '202412' for now" @@ -4418,11 +4500,9 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiDot1pQueueMapping: qos/test_qos_sai.py::TestQosSai::testQosSaiDscpEcn: skip: - reason: "Unsupported testbed type or test fails on all topologies. https://github.com/sonic-net/sonic-mgmt/issues/23059" - conditions_logical_operator: or + reason: "Unsupported testbed type" conditions: - - "asic_type not in ['broadcom'] and asic_subtype not in ['broadcom-dnx']" - - "https://github.com/sonic-net/sonic-mgmt/issues/23059" + - "platform not in ['x86_64-nokia_ixr7250e_36x400g-r0']" qos/test_qos_sai.py::TestQosSai::testQosSaiDscpQueueMapping: skip: @@ -4649,11 +4729,19 @@ radv/test_radv_ipv6_ra.py::test_unsolicited_router_advertisement_with_m_flag: ####################################### ##### read_mac ##### ####################################### +read_mac: + skip: + reason: "Read_mac tests are not supported on lossy topologies." + conditions: + - *lossyTopos + read_mac/test_read_mac_metadata.py: skip: reason: "Read_mac test needs specific variables and image urls, currently do not support on KVM and regular nightly test." + conditions_logical_operator: or conditions: - "asic_type in ['vs']" + - *lossyTopos ####################################### ##### reboot ##### @@ -4667,6 +4755,12 @@ reboot/test_reboot_blocking_mode.py: ####################################### ##### reset_factory ##### ####################################### +reset_factory: + skip: + reason: "Reset factory tests are not supported on lossy topologies." + conditions: + - *lossyTopos + reset_factory/test_reset_factory.py: skip: reason: "This case has a known issue which leaves the DUT in a bad state. Skipping until the issue is addressed. Also skipped on BMC due to a system clock image issue." @@ -4674,6 +4768,7 @@ reset_factory/test_reset_factory.py: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/11103 - "'bmc' in topo_type" + - *lossyTopos ####################################### ##### restapi ##### @@ -4846,12 +4941,24 @@ route/test_static_route.py::test_static_route_ecmp_ipv6: - "release in ['201811', '201911']" - "'standalone' in topo_name" +route/test_static_route.py::test_static_route_ecmp_warmboot: + skip: + reason: "Dualtor topology doesn't support warm reboot on mellanox platform" + conditions: + - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120', 'dualtor-aa', 'dualtor-aa-56', 'dualtor-aa-64-breakout'] and asic_type in ['mellanox', 'nvidia']" + route/test_static_route.py::test_static_route_ipv6: xfail: reason: "Test case has issue on the t0-isolated-d256u256s2 topo." conditions: - "'t0-isolated-d256u256s2' in topo_name and platform in ['x86_64-nvidia_sn5640-r0']" +route/test_static_route.py::test_static_route_ipv6_warmboot: + skip: + reason: "Dualtor topology doesn't support warm reboot on mellanox platform" + conditions: + - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120', 'dualtor-aa', 'dualtor-aa-56', 'dualtor-aa-64-breakout'] and asic_type in ['mellanox', 'nvidia']" + ####################################### ##### sai_qualify ##### ####################################### @@ -4963,14 +5070,25 @@ show_techsupport/test_techsupport.py::test_techsupport: - "https://github.com/sonic-net/sonic-mgmt/issues/21690" +####################################### +##### smartswitch ##### +####################################### +smartswitch: + skip: + reason: "Smartswitch tests are not supported on lossy topologies." + conditions: + - *lossyTopos + ####################################### ##### snappi_tests ##### ####################################### snappi_tests: skip: - reason: "Snappi test only support on physical tgen testbed" + reason: "Snappi test only support on physical tgen testbed and is not supported on lossy topologies" + conditions_logical_operator: or conditions: - "asic_type in ['vs']" + - *lossyTopos snappi_tests/bgp/test_bgp_convergence_performance.py: xfail: @@ -5090,11 +5208,19 @@ snappi_tests/reboot: ##### snmp ##### ####################################### +snmp: + skip: + reason: "SNMP tests are not supported on lossy topologies." + conditions: + - *lossyTopos + snmp/: skip: reason: 'SNMP tests may depend on ASIC-specific MIBs, not available on BMC' + conditions_logical_operator: or conditions: - "'bmc' in topo_type" + - *lossyTopos snmp/test_snmp_default_route.py: xfail: @@ -5197,15 +5323,19 @@ span/test_port_mirroring.py: ####################################### srv6/test_srv6_basic_sanity.py: skip: - reason: "It's a new test case, skip it for other topologies except cisco vs nodes and force nodes." + reason: "It's a new test case, skip it for other topologies except cisco vs nodes and force nodes. Skip for t1-isolated-d32/128" + conditions_logical_operator: or conditions: - topo_name not in ["ciscovs-7nodes", "ciscovs-5nodes", "force10-7nodes"] + - "topo_name in ['t1-isolated-d128', 't1-isolated-d32']" srv6/test_srv6_basic_sanity.py::test_traffic_check_normal: skip: - reason: "It's a new test case, skip it for other topologies except cisco vs nodes (SRv6 data plane required)." + reason: "It's a new test case, skip it for other topologies except cisco vs nodes (SRv6 data plane required). Skip for t1-isolated-d32/128" + conditions_logical_operator: or conditions: - topo_name not in ["ciscovs-7nodes", "ciscovs-5nodes"] + - "topo_name in ['t1-isolated-d128', 't1-isolated-d32']" srv6/test_srv6_dataplane.py: skip: @@ -5551,6 +5681,7 @@ telemetry/test_telemetry_srv6.py: conditions_logical_operator: or conditions: - "asic_type == 'broadcom' and asic_gen not in ['th5', 'th6', 'q3d']" + - "asic_type == 'mellanox' and asic_gen in ['spc1', 'spc2', 'spc3']" - "'bmc' in topo_type" telemetry/test_telemetry_srv6.py::test_poll_mode_srv6_sid_counters: @@ -5580,9 +5711,11 @@ test_nbr_health.py: ####################################### test_pktgen.py: skip: - reason: "No need in M0/Mx/M1" + reason: "No need in M0/Mx/M1 or lossy topologies" + conditions_logical_operator: or conditions: - "topo_type in ['bmc', 'm0', 'm1', 'mx']" + - *lossyTopos ####################################### ##### pretest ##### @@ -5598,9 +5731,23 @@ test_pretest.py::test_disable_rsyslog_rate_limit: ####################################### test_vs_chassis_setup.py: skip: - reason: "Skip vs_chassis setup on non-vs testbeds" + reason: "Skip vs_chassis setup on non-vs testbeds and lossy topologies" + conditions_logical_operator: or conditions: - "asic_type not in ['vs']" + - *lossyTopos + +tests/ha/test_ha_bfd_pin.py: + xfail: + reason: "BFD pin test is expected to fail on Cisco 8102 SmartSwitch with SONiC 202511" + conditions: + - "is_smartswitch == True and platform in ['x86_64-8102_28fh_dpu_o-r0'] and branch in ['202511', 'internal-202511']" + +tests/ha/test_ha_repairing_dpu.py: + xfail: + reason: "HA repairing DPU test is expected to fail on Cisco 8102 SmartSwitch with SONiC 202511" + conditions: + - "is_smartswitch == True and platform in ['x86_64-8102_28fh_dpu_o-r0'] and branch in ['202511', 'internal-202511']" ####################################### ##### upgrade_path ##### @@ -5611,10 +5758,12 @@ upgrade_path: Skipped due to one or more unsupported conditions: - Upgrade path test needs base and target image lists, currently do not support on KVM. - Not supported on t1 topology + - Not supported on lossy topologies conditions_logical_operator: or conditions: - "asic_type in ['vs']" - "'t1' in topo_type and asic_type in ['marvell-teralynx']" + - *lossyTopos upgrade_path/test_multi_hop_upgrade_path.py: skip: @@ -5667,11 +5816,12 @@ vlan/test_vlan_ping.py: ####################################### voq: skip: - reason: "Cisco 8800 doesn't support voq tests and not supported on this DUT topology" + reason: "Cisco 8800 doesn't support voq tests and not supported on this DUT topology or lossy topologies" conditions_logical_operator: or conditions: - "asic_type in ['cisco-8000']" - "'t2' not in topo_name and topo_type not in ['lrh', 'urh']" + - *lossyTopos voq/test_fabric_cli_and_db.py: skip: @@ -5975,9 +6125,19 @@ vxlan/test_vxlan_decap.py: vxlan/test_vxlan_decap_ttl.py: skip: - reason: "VxLAN tunnel TTL decap mode test is not supported on multi-ASIC platform. Also this test can only run and currently passes only on some platforms." + reason: "VxLAN tunnel TTL decap mode test is not supported on multi-ASIC platforms and dualtor topologies. Also this test can only run and currently passes only on some platforms." + conditions_logical_operator: OR + conditions: + - "is_multi_asic == True" + - "platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0', 'x86_64-8102_28fh_dpu_o-r0']" + - "'dualtor' in topo_name" + +vxlan/test_vxlan_decap_ttl.py::test_vxlan_decap_ttl\[v6-.*\]: + regex: true + skip: + reason: "IPv6 VXLAN tunnels are not supported on Mellanox Spectrum-1." conditions: - - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0', 'x86_64-8102_28fh_dpu_o-r0'])" + - "asic_type in ['mellanox'] and asic_gen == 'spc1'" vxlan/test_vxlan_ecmp.py: skip: @@ -6073,6 +6233,15 @@ vxlan/test_vxlan_vnet_bgp_subintf.py: - "asic_type in ['cisco-8000']" - "release in ['202511']" +####################################### +##### wan ##### +####################################### +wan: + skip: + reason: "WAN tests are not supported on lossy topologies." + conditions: + - *lossyTopos + ####################################### ##### wan_lacp ##### ####################################### @@ -6089,9 +6258,20 @@ wan/lacp/test_wan_lag_min_link.py::test_lag_min_link: ####################################### wol: skip: - reason: "Not supported on this DUT topology" + reason: "Not supported on this DUT topology or lossy topologies" + conditions_logical_operator: or conditions: - "topo_type not in ['mx', 'm0']" + - *lossyTopos + +####################################### +##### wred ##### +####################################### +wred/test_wred_counters.py: + skip: + reason: "WRED/ECN is not supported on KVM switches." + conditions: + - "asic_type in ['vs']" ####################################### ##### zmq ##### diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_live_addon_docker.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_live_addon_docker.yaml new file mode 100644 index 00000000000..790dd3c28c4 --- /dev/null +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_live_addon_docker.yaml @@ -0,0 +1,11 @@ +####################################### +##### live addon docker ##### +####################################### + +live_addon_docker/test_live_addon_docker.py: + skip: + reason: "Live-addon docker tests require vendor JSON for this asic_type (skip VS/KVM and other platforms)" + conditions_logical_operator: or + conditions: + - "asic_type not in ['cisco-8000']" + - "asic_type in ['vs']" diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml index a6405beb161..f2657e7eb2a 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml @@ -983,6 +983,7 @@ platform_tests/cli/test_show_platform.py::test_show_platform_fan: reason: "Unsupported platform API" conditions: - "is_multi_asic==True and release in ['201911']" + - "'bmc' in topo_type" platform_tests/cli/test_show_platform.py::test_show_platform_firmware_status: skip: @@ -1002,6 +1003,7 @@ platform_tests/cli/test_show_platform.py::test_show_platform_psustatus: conditions: - "'nvda_bf' in platform" - "platform in ['arm64-elba-asic-flash128-r0']" + - "'bmc' in topo_type" platform_tests/cli/test_show_platform.py::test_show_platform_psustatus_json: xfail: @@ -1015,6 +1017,7 @@ platform_tests/cli/test_show_platform.py::test_show_platform_psustatus_json: conditions: - "'nvda_bf' in platform" - "platform in ['arm64-elba-asic-flash128-r0']" + - "'bmc' in topo_type" platform_tests/cli/test_show_platform.py::test_show_platform_syseeprom: xfail: @@ -1154,8 +1157,10 @@ platform_tests/mellanox/test_hw_management_service.py: platform_tests/mellanox/test_psu_power_threshold.py: skip: - reason: "Not available on BMC" + reason: "Test is skipped on Mellanox LD platform for it doesn't have PSU Power Threshold, or not available for BMC" + conditions_logical_operator: or conditions: + - "asic_type in ['mellanox'] and 'sn6600_ld' in platform" - "'bmc' in topo_type" platform_tests/mellanox/test_reboot_cause.py: @@ -1440,9 +1445,11 @@ platform_tests/test_port_toggle.py: platform_tests/test_power_off_reboot.py: skip: - reason: "Skip power off reboot test for Wistron/Nokia-7215" + reason: "Skip power off reboot test for Wistron/Nokia-7215, or no PSU for BMC" + conditions_logical_operator: or conditions: - "(hwsku in ['Celestica-E1031-T48S4']) or ('sw_to3200k' in hwsku) or (platform in ['armhf-nokia_ixs7215_52x-r0']) or (is_multi_asic==True and release in ['201911'])" + - "'bmc' in topo_type" platform_tests/test_reboot.py::test_cold_reboot: xfail: @@ -1565,6 +1572,12 @@ platform_tests/test_sensors.py::test_sensors: ####################################### ##### test_sequential_restart.py ##### ####################################### +platform_tests/test_sequential_restart.py::test_restart_swss: + skip: + reason: "BMC does not support swss daemon" + conditions: + - "'bmc' in topo_type" + platform_tests/test_sequential_restart.py::test_restart_syncd: skip: reason: "Restarting syncd is not supported yet" diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_sonic_vpp.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_sonic_vpp.yaml index 3f2b9e41cf7..8c4f573e0f3 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_sonic_vpp.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_sonic_vpp.yaml @@ -156,14 +156,6 @@ copp: ####################################### ##### Decap ##### ####################################### -decap/test_decap.py: - skip: - reason: > - Failed/Errored: To be included - conditions_logical_operator: or - conditions: - - "asic_type in ['vpp']" - decap/test_subnet_decap.py: skip: reason: > diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t1_multiasic.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t1_multiasic.yaml index 0fb3524e06b..9db56ebb8c0 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t1_multiasic.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t1_multiasic.yaml @@ -18,11 +18,6 @@ bgp/test_bgp_allow_list.py: conditions: - asic_type in ['vs'] and 't1-8-lag' in topo_name reason: This test case either cannot pass or should be skipped on virtual chassis -bgp/test_bgp_bbr_default_state.py: - skip: - conditions: - - asic_type in ['vs'] and 't1-8-lag' in topo_name - reason: This test case either cannot pass or should be skipped on virtual chassis bgp/test_bgp_bounce.py: skip: conditions: diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index b9682b62e9e..ba72e2fc669 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -1395,6 +1395,18 @@ def check_disk_usage(duthosts): DISK_USAGE_THRESHOLD = 90 # percentage - aligned with monit resource limit + # Pseudo/virtual filesystems are not backed by real storage, are not + # monitored by monit, and routinely report misleading usage. For example + # efivarfs (mounted at /sys/firmware/efi/efivars) exposes a tiny UEFI + # variable store that frequently sits at ~99% and is unrelated to disk + # health. Skip these filesystem types to avoid false-positive failures. + SKIP_FSTYPES = frozenset([ + "efivarfs", "tmpfs", "devtmpfs", "devpts", "sysfs", "proc", + "cgroup", "cgroup2", "overlay", "squashfs", "ramfs", "debugfs", + "tracefs", "securityfs", "pstore", "mqueue", "hugetlbfs", + "configfs", "fusectl", "bpf", "autofs", "binfmt_misc", + ]) + def _check(*args, **kwargs): init_result = {"failed": False, "check_item": "disk_usage"} result = parallel_run(_check_disk_usage_on_dut, args, kwargs, duthosts, @@ -1408,7 +1420,7 @@ def _check_disk_usage_on_dut(*args, **kwargs): logger.info("Checking disk usage on %s..." % dut.hostname) check_result = {"failed": False, "check_item": "disk_usage", "host": dut.hostname} - res = dut.shell("df --output=pcent,target,source", module_ignore_errors=True) + res = dut.shell("df --output=pcent,target,source,fstype", module_ignore_errors=True) if res["rc"] != 0: logger.error("Failed to get disk usage on %s: %s" % (dut.hostname, res.get("stderr", ""))) check_result["failed"] = True @@ -1420,9 +1432,9 @@ def _check_disk_usage_on_dut(*args, **kwargs): line = line.strip() if not line: continue - # Format: "Use% Mounted on Filesystem" e.g. " 92% / /dev/sda3" - parts = line.split(None, 2) - if len(parts) < 2: + # Format: "Use% Mounted on Filesystem Type" e.g. " 92% / /dev/sda3 ext4" + parts = line.split(None, 3) + if len(parts) < 3: continue usage_str = parts[0].rstrip('%') try: @@ -1430,7 +1442,12 @@ def _check_disk_usage_on_dut(*args, **kwargs): except ValueError: continue mount_point = parts[1] - filesystem = parts[2] if len(parts) > 2 else "" + filesystem = parts[2] + fstype = parts[3] if len(parts) > 3 else "" + # Skip pseudo/virtual filesystems (e.g. efivarfs, tmpfs) that are + # not real storage and would otherwise cause false positives. + if fstype in SKIP_FSTYPES: + continue if usage_pct >= DISK_USAGE_THRESHOLD: over_threshold.append({ "mount": mount_point, diff --git a/tests/common/snappi_tests/ixload/snappi_fixtures.py b/tests/common/snappi_tests/ixload/snappi_fixtures.py index 9a7934306ef..3d008c5d63e 100755 --- a/tests/common/snappi_tests/ixload/snappi_fixtures.py +++ b/tests/common/snappi_tests/ixload/snappi_fixtures.py @@ -3,7 +3,8 @@ """ from tests.common.snappi_tests.ixload.snappi_helper import (l47_trafficgen_main, duthost_ha_config, npu_startup, dpu_startup, set_static_routes, set_ha_roles, - set_ha_admin_up, set_ha_activate_role, duthost_port_config) + set_ha_admin_up, set_ha_activate_role, duthost_port_config, + delete_staticarp_files) from tests.common.snappi_tests.uhd.uhd_helpers import NetworkConfigSettings # noqa: F403, F401 import pytest import threading @@ -78,15 +79,22 @@ def setup_config_snappi_l47(request, duthosts, tbinfo, ha_test_case=None): if l47_trafficgen_enabled: logger.info(f"Configuring L47 parameters for test case: {ha_test_case}") + ixos_version = tbinfo['ixos_version'] l47_version = tbinfo['l47_version'] service_type = tbinfo['service_type'] chassis_ip = tbinfo['chassis_ip'] + clean_l47trafficgen_staticarps = tbinfo['clean_l47trafficgen_staticarps'] gw_ip = tbinfo['l47_gateway'] + eni_per_dpu = int(tbinfo.get('eni_per_dpu', 32)) ports_list = tbinfo['ports_list'] ports_list = {k: [tuple(x) for x in v] for k, v in ports_list.items()} - test_filename = "dash_cps" + if service_type == "vnet2vnet": + test_filename = "dash_vnet2vnet" + else: + test_filename = "dash_pl" + initial_cps_obj = (len(ports_list['Traffic1@Network1']) * 4000000) // 2 test_type_dict = { @@ -98,14 +106,23 @@ def setup_config_snappi_l47(request, duthosts, tbinfo, ha_test_case=None): 'chassis_ip': chassis_ip, 'gw_ip': gw_ip, 'port': '8080', + 'ixos_version': ixos_version, 'version': l47_version, } + logger.info("Cleaning old static ARP files from the l47traifficgen server") + if clean_l47trafficgen_staticarps: + delete_staticarp_files(chassis_ip, tbinfo, ixos_version) + nw_config = NetworkConfigSettings() if ha_test_case != "cps": nw_config.ENI_COUNT = 32 # Set to 32 ENIs for HA test cases to test 1 Active/Standby DPU - api, config, initial_cps_value = l47_trafficgen_main(ports_list, connection_dict, nw_config, service_type, - test_type_dict['all'], test_type_dict['initial_cps_obj']) + if ha_test_case == "cps" and eni_per_dpu > 0: + nw_config.ENI_COUNT = eni_per_dpu + api, config, initial_cps_value = l47_trafficgen_main(ports_list, tbinfo, connection_dict, nw_config, + service_type, + test_type_dict['all'], test_filename, + test_type_dict['initial_cps_obj']) if l47_trafficgen_save: snappi_l47_params['save'] = True @@ -247,7 +264,7 @@ def run_dpu_startup(duthosts, duthost, tbinfo, static_ipsmacs_dict, ha_test_case duthost1 = duthosts[0] # Configure SmartSwitch - # duthost_port_config(duthost) + duthost_port_config(duthost1) static_ipsmacs_dict1 = duthost_ha_config(duthost1, nw_config) @@ -281,6 +298,7 @@ def config_snappi_l47(request, duthosts, tbinfo): """ Fixture configures L47 parameters """ + logger.info("Entering L47 configuration setup") return setup_config_snappi_l47(request, duthosts, tbinfo) diff --git a/tests/common/snappi_tests/ixload/snappi_helper.py b/tests/common/snappi_tests/ixload/snappi_helper.py index 99345f7e00d..2d635ceda47 100644 --- a/tests/common/snappi_tests/ixload/snappi_helper.py +++ b/tests/common/snappi_tests/ixload/snappi_helper.py @@ -3,8 +3,11 @@ from tests.common.snappi_tests.uhd.uhd_helpers import NetworkConfigSettings # noqa: F403, F401 from concurrent.futures import ThreadPoolExecutor, as_completed from netmiko import ConnectHandler +from scp import SCPClient from pathlib import Path +import paramiko import snappi +import stat import time import os import glob @@ -16,6 +19,42 @@ logger = logging.getLogger(__name__) +def patch_dutnetwork_range(api, base_url, session_id, test_id, dut_id, network_range_id, first_ip, username, password): + """ + Sends a PATCH request to update the first IP in the network range. + + Args: + base_url (str): The base URL of the API (e.g., "https://10.3.8.23:8443"). + session_id (int): The session ID (e.g., 0). + test_id (str): The test ID (e.g., "activeTest"). + dut_id (int): The DUT ID (e.g., 0). + network_range_id (int): The network range ID (e.g., 0). + first_ip (str): The new first IP to set (e.g., "1.2.3.4"). + username (str): The username for authentication. + password (str): The password for authentication. + + Returns: + Response: The response object from the PATCH request. + """ + url = (f"{base_url}/api/v1/sessions/{session_id}/ixload/test/{test_id}/dutList/{dut_id}/dutConfig/networkRangeList/" + f"{network_range_id}") + payload = { + "firstIp": first_ip + } + + try: + res = api.ixload_configure("patch", url, payload) + if res.status_code == 204: + logger.info("PATCH request successful: 204 No Content") + else: + logger.info(f"PATCH request failed: {res.status_code} {res.reason}") + logger.info(res.text) + return res + except res.RequestException as e: + logger.info(f"An error occurred: {e}") + return None + + def set_static_routes(duthost, static_ipmacs_dict): static_macs = static_ipmacs_dict['static_macs'] @@ -196,8 +235,8 @@ def set_ha_admin_up(duthosts, duthost, tbinfo): logger.info(f"Active side cmd1 output: {output_cmd1['stdout']}") time.sleep(2) - output = duthost.shell(f'sudo arp -s {standby_ethpass_ip} {standby_mac}') - output_ping = duthost.command(f"ping -c 3 {standby_ethpass_ip}", module_ignore_errors=True) # noqa: F841 + duthost.shell(f'sudo arp -s {standby_ethpass_ip} {standby_mac}') + duthost.command(f"ping -c 3 {standby_ethpass_ip}", module_ignore_errors=True) except Exception as e: logger.error(f"{duthost.hostname} Error setting HA admin up active side: {str(e)}") else: @@ -212,8 +251,8 @@ def set_ha_admin_up(duthosts, duthost, tbinfo): logger.info(f"Standby side cmd1 output: {output_cmd1['stdout']}") time.sleep(2) - output = duthost.shell(f'sudo arp -s {active_ethpass_ip} {active_mac}') # noqa: F841 - output_ping = duthost.command(f"ping -c 3 {active_ethpass_ip}", module_ignore_errors=True) # noqa: F841 + duthost.shell(f'sudo arp -s {active_ethpass_ip} {active_mac}') + duthost.command(f"ping -c 3 {active_ethpass_ip}", module_ignore_errors=True) except Exception as e: logger.error(f"{duthost.hostname} Error setting HA admin up standby side: {str(e)}") @@ -307,13 +346,13 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in if duthost == duthosts[1]: target_ip = f'169.254.200.{dpu_index + 1}' else: - target_ip = f'18.{dpu_index}.202.1' + target_ip = f'20.0.200.{dpu_index+1}' else: - target_ip = f'18.{dpu_index}.202.1' + target_ip = f'20.0.200.{dpu_index+1}' else: - target_ip = f'18.{dpu_index}.202.1' - target_username = 'admin' - target_password = 'YourPaSsWoRd' + target_ip = f'20.0.200.{dpu_index+1}' + target_username = tbinfo.get('dpu_target_username', 'admin') + target_password = tbinfo.get('dpu_target_passwd', 'YourPaSsWoRd') # Connect to jump host net_connect_jump = ConnectHandler(**jump_host) @@ -361,9 +400,9 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in 'sudo config route del prefix 0.0.0.0/0 via 169.254.200.254', delay_factor=2) logger.info(f"{duthost.hostname} Execute on DPU Target: {output}") time.sleep(1) - logger.info(f'sudo config route add prefix 0.0.0.0/0 nexthop 18.{dpu_index}.202.0') + logger.info(f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.200.{dpu_index+1}') output = net_connect_jump.send_command_timing( - f'sudo config route add prefix 0.0.0.0/0 nexthop 18.{dpu_index}.202.0', delay_factor=2) + f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.200.{dpu_index+1}', delay_factor=2) logger.info(f"{duthost.hostname} Execute on DPU Target: {output}") output = net_connect_jump.send_command_timing('show ip route', delay_factor=2) logger.info(f"{duthost.hostname} Execute on DPU Target: {output}") @@ -409,10 +448,10 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in logger.info(f'Restarting hamgrd on {duthost.hostname}: docker restart dash-hadpu0') duthost.shell("docker restart dash-hadpu0") logger.info(f'Removing interface from {duthost.hostname}: ' - f'sudo config interface ip rem Ethernet0 18.{dpu_index}.202.1/31') + f'sudo config interface ip rem Ethernet0 20.0.200.{dpu_index+1}/31') time.sleep(2) output = net_connect_jump.send_command_timing( - f'sudo config interface ip rem Ethernet0 18.{dpu_index}.202.1/31', delay_factor=2) + f'sudo config interface ip rem Ethernet0 20.0.200.{dpu_index+1}/31', delay_factor=2) logger.info( f'Deleting route on {duthost.hostname}: sudo ip route del 0.0.0.0/0 via 169.254.200.254') time.sleep(2) @@ -420,10 +459,10 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in 'sudo ip route del 0.0.0.0/0 via 169.254.200.254', delay_factor=2) logger.info( f'Adding route on {duthost.hostname}: ' - f'sudo config route add prefix 0.0.0.0/0 nexthop 20.{dpu_index}.202.0') + f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.201.{dpu_index+1}') time.sleep(2) output = net_connect_jump.send_command_timing( - f'sudo config route add prefix 0.0.0.0/0 nexthop 20.{dpu_index}.202.0', delay_factor=2) + f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.201.{dpu_index+1}', delay_factor=2) active_ethpass_ip = tbinfo['active_ethpass_ip'] active_mac = tbinfo['active_mac'] logger.info( @@ -432,11 +471,11 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in logger.info( f'Pinging standby side loopback intf from {duthost.hostname}: ' f'ping -c 3 {active_ethpass_ip}') # noqa: E231 - output_ping = duthost.command(f"ping -c 3 {active_ethpass_ip}", module_ignore_errors=True) + duthost.command(f"ping -c 3 {active_ethpass_ip}", module_ignore_errors=True) logger.info(f'Correcting route on {duthost.hostname}: ' - f'sudo config route del prefix 221.0.0.{dpu_index+1}/32 nexthop 20.{dpu_index}.202.1') + f'sudo config route del prefix 221.0.0.{dpu_index+1}/32 nexthop 20.0.201.{dpu_index+1}') output = duthost.command(f'sudo config route del prefix 221.0.0.{dpu_index+1}/32 ' - f'nexthop 20.{dpu_index}.202.1', module_ignore_errors=True) + f'nexthop 20.0.201.{dpu_index+1}', module_ignore_errors=True) logger.info(f'Adding correct route on {duthost.hostname}: ' f'sudo config route add prefix 221.0.0.{dpu_index+1}/32 nexthop 220.0.4.1') output = duthost.command(f'sudo config route add prefix 221.0.0.{dpu_index+1}/32 ' @@ -452,10 +491,10 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in output = net_connect_jump.send_command_timing( 'sudo ip route del 0.0.0.0/0 via 169.254.200.254', delay_factor=2) logger.info(f'Adding route on {duthost.hostname}: ' - f'sudo config route add prefix 0.0.0.0/0 nexthop 18.{dpu_index}.202.0') + f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.200.{dpu_index+1}') time.sleep(2) output = net_connect_jump.send_command_timing( - f'sudo config route add prefix 0.0.0.0/0 nexthop 18.{dpu_index}.202.0', delay_factor=2) + f'sudo config route add prefix 0.0.0.0/0 nexthop 20.0.200.{dpu_index+1}', delay_factor=2) standby_ethpass_ip = tbinfo['standby_ethpass_ip'] standby_mac = tbinfo['standby_mac'] @@ -464,8 +503,7 @@ def _set_routes_on_dut(duthosts, duthost, tbinfo, local_files, local_dir, dpu_in output = duthost.shell(f'sudo arp -s {standby_ethpass_ip} {standby_mac}') logger.info( f'Pinging active side loopback intf from {duthost.hostname}: sudo ping -c 3 {standby_ethpass_ip}') # noqa: E231 - output_ping = duthost.command(f"ping -c 3 {standby_ethpass_ip}", # noqa: F841 - module_ignore_errors=True) + duthost.command(f"ping -c 3 {standby_ethpass_ip}", module_ignore_errors=True) except Exception as e: logger.error(f"{duthost.hostname} Error during DPU configuration: {str(e)}") raise @@ -536,7 +574,7 @@ def load_dpu_configs_on_dut( ha_test_case) remote_dir = f"{remote_dir}/dpu{dpu_index}" _ensure_remote_dir_on_dut(duthost, remote_dir) - _telemetry_run_on_dut(duthost) + # _telemetry_run_on_dut(duthost) _copy_files_to_dut(duthost, local_files, remote_dir) delay = initial_delay_sec @@ -551,7 +589,7 @@ def load_dpu_configs_on_dut( logger.info(f"RPC unavailable for {rb}; retrying after telemetry restart") # noqa: E702 duthost.shell("docker ps --format '{{.Names}}' | grep -w gnmi || true", module_ignore_errors=True) time.sleep(120) - _telemetry_run_on_dut(duthost) + # _telemetry_run_on_dut(duthost) time.sleep(retry_delay_sec) _docker_run_config_on_dut(duthost, remote_dir, dpu_index, rb) @@ -564,14 +602,11 @@ def load_dpu_configs_on_dut( def duthost_port_config(duthost): # copy HA config - # duthost.command("sudo cp {} {}".format( - # "/etc/sonic/0HA_BACKUP/config_db.json", "/etc/sonic/config_db.json")) logger.info(f"{duthost.hostname} Loading custom HA config_db.json") duthost.shell("sudo sonic-cfggen -j /etc/sonic/0HA_BACKUP/config_db.json --write-to-db") duthost.shell("sudo cp /etc/sonic/0HA_BACKUP/config_db.json /etc/sonic/config_db.json") # logger.info(f"{duthost.hostname} Reloading config_db.json") - # duthost.shell("sudo config reload -y \n") logger.info(f"{duthost.hostname} Saving config_db.json") duthost.shell("sudo config save -y") @@ -582,14 +617,6 @@ def duthost_port_config(duthost): def duthost_ha_config(duthost, nw_config): # Smartswitch configure - """ - logger.info('Cleaning up config') - logger.info("Wait until all critical services are fully started") - pytest_assert(wait_until(360, 10, 1, - duthost.critical_services_fully_started), - "Not all critical services are fully started") - - """ static_ipsmacs_dict = {} @@ -856,6 +883,7 @@ def npu_startup(duthosts, duthost, localhost): logger.info(f"DPU boot successful on {duthost.hostname}") break + ''' if duthost == duthosts[1]: # standby device logger.info(f"Removing unwanted IPs from standby device: {duthost.hostname}") remove_dpu_ip_addresses_from_npu( @@ -863,6 +891,7 @@ def npu_startup(duthosts, duthost, localhost): ip_prefixes_to_remove=["18"], # Removes 18.X.202.0/31 addresses additional_filters=["220.0.1.1/", "220.0.2.1/", "220.0.3.1/", "220.0.4.1/"] ) + ''' return True @@ -870,33 +899,18 @@ def npu_startup(duthosts, duthost, localhost): def dpu_startup(duthosts, duthost, tbinfo, static_ipmacs_dict, ha_test_case): logger.info(f"Pinging each DPU on {duthost.hostname}") - """ - dpuIFKeys = [k for k in static_ipmacs_dict['static_ips'] if k.startswith("221.0")] - passing_dpus = [] - - for x, ipKey in enumerate(dpuIFKeys): - logger.info(f"On {duthost.hostname} pinging DPU{x}: {static_ipmacs_dict['static_ips'][ipKey]}") - output_ping = duthost.command(f"ping -c 3 {static_ipmacs_dict['static_ips'][ipKey]}", module_ignore_errors=True) - if output_ping.get("rc", 1) == 0 and "0% packet loss" in output_ping.get("stdout", ""): - logger.info(f"Ping success on {duthost.hostname}") - passing_dpus.append(x) - pass - else: - logger.info(f"Ping failure on {duthost.hostname}") - pass - """ remote_dir = "/tmp/dpu_configs" initial_delay_sec = 20 retry_delay_sec = 10 # Determine which IPs to ping based on duthost if len(duthosts) > 1 and duthost == duthosts[1]: - # For duthosts[1], ping 20.0.202.1, 20.1.202.1, ..., 20.7.202.1 + # For duthosts[1], ping 20.0.201.1...20.0.201.8 ip_list_to_ping = [f"169.254.200.{i+1}" for i in range(8)] logger.info(f"Using standby side midplane IPs for {duthost.hostname}: {ip_list_to_ping}") elif len(duthosts) > 1 and duthost == duthosts[0]: - # For duthosts[0], ping 18.0.202.1, ..., 18.7.202.1 - ip_list_to_ping = [f"18.{i}.202.1" for i in range(8)] + # For duthosts[0], ping 20.0.200.1, ..., 20.0.200.8 + ip_list_to_ping = [f"20.0.200.{i+1}" for i in range(8)] logger.info(f"Using active side IPs for {duthost.hostname}: {ip_list_to_ping}") else: # Fallback to original logic for single DUT setup @@ -919,19 +933,6 @@ def dpu_startup(duthosts, duthost, tbinfo, static_ipmacs_dict, ha_test_case): errors = {} - """ - if ha_test_case != "cps": - max_workers = 2 - required_dpus = [0, 2] - if all(dpu in passing_dpus for dpu in required_dpus): - passing_dpus = required_dpus - else: - passing_dpus = [] - return passing_dpus - else: - max_workers = min(8, max(1, len(passing_dpus))) - """ - # max_workers = min(8, max(1, len(passing_dpus))) max_workers = min(8, max(1, len(passing_dpus))) logger.info("{} DPU config loading DPUs, passing_dpus: {}".format(duthost.hostname, passing_dpus)) @@ -996,32 +997,81 @@ def assignPorts(api, ports_list): chassisId, cardId, portId = portTuple paramDict = {"chassisId": chassisId, "cardId": cardId, "portId": portId} try: - # Code that may raise an exception - res = api.ixload_configure("post", portListUrl, paramDict) # noqa: F841 + api.ixload_configure("post", portListUrl, paramDict) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return -def build_node_ips(count, vpc, nw_config, nodetype="client"): - if nodetype in "client": - ip = nw_config.ipp(int(nw_config.IP_R_START) + (nw_config.IP_STEP_NSG * count) - + int(nw_config.IP_STEP_ENI) * (vpc - 1)) - if nodetype in "server": - ip = nw_config.ipp(int(nw_config.IP_L_START) + int(nw_config.IP_STEP_ENI) * (vpc - 1)) +def checkPorts(api, ports_list): + + # Expected port tuples + expected_ports = [port for ports in ports_list.values() for port in ports] + logger.info(f"Expected ports: {expected_ports}") + + # Check both client and server + for objectID, url in [(0, "ixload/test/activeTest/communityList/0/network/portList"), + (1, "ixload/test/activeTest/communityList/1/network/portList")]: + port_ids = get_objectIDs(api, url) + logger.info(f"Community {objectID} portListIDs: {port_ids}") + + for port_id in port_ids: + payload = {} + port_details = api.ixload_configure("get", f"{url}/{port_id}", payload) + port_tuple = (port_details.chassisId, + port_details.cardId, + port_details.portId) + + if port_tuple not in expected_ports: + logger.info(f"Removing port {port_tuple} (objectID: {port_id})") + api.ixload_configure("delete", f"{url}/{port_id}", {}) + + return + + +def build_node_ips(count, vpc, nw_config, service_type='vnet2vnet', nodetype="client"): + + if service_type == 'vnet2vnet': + if nodetype == "client": + ip = nw_config.ipp(int(nw_config.IP_R_START) + (nw_config.IP_STEP_NSG * count) + + int(nw_config.IP_STEP_ENI) * (vpc - 1)) + if nodetype == "server": + ip = nw_config.ipp(int(nw_config.IP_L_START) + int(nw_config.IP_STEP_ENI) * (vpc - 1)) + else: + # service_type == 'privatelink' + if nodetype == "dut_client": + ip = nw_config.ipp(int(nw_config.IP_R_START) + (nw_config.IP_STEP_NSG * count) + + int(nw_config.IP_STEP_ENI) * (vpc - 1)) + if nodetype == "client": + ip = nw_config.ipp(int(nw_config.IP_L_START) + int(nw_config.IP_STEP_ENI) * (vpc - 1)) + if nodetype == "server": + ip = nw_config.ipp((int(nw_config.IPv6_R_START) + (int(nw_config.IPv6_Range_Increment) * (vpc - 1)))) return str(ip) -def build_node_macs(count, vpc, nw_config, nodetype="client"): +def build_node_macs(count, vpc, nw_config, service_type='vnet2vnet', nodetype="client"): - if nodetype in "client": - m = nw_config.maca(int(nw_config.MAC_R_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1) - + (int(nw_config.maca(nw_config.ACL_TABLE_MAC_STEP)) * count)) - if nodetype in "server": - m = nw_config.maca(int(nw_config.MAC_L_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1)) + if service_type == 'vnet2vnet': + if nodetype == "client": + m = nw_config.maca(int(nw_config.MAC_R_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1) + + (int(nw_config.maca(nw_config.ACL_TABLE_MAC_STEP)) * count)) + if nodetype == "server": + m = nw_config.maca(int(nw_config.MAC_L_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1)) + else: + # service_type == private_link + if nodetype == "staticarp_client": + m = nw_config.maca(int(nw_config.MAC_R_START) + int(nw_config.maca('00:00:00:00:00:80')) * (vpc - 1) + + (int(nw_config.maca(nw_config.ENI_MAC_STEP)) * count)) + if nodetype == "dut_client": + m = nw_config.maca(int(nw_config.MAC_R_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1) + + (int(nw_config.maca(nw_config.ACL_TABLE_MAC_STEP)) * count)) + if nodetype == "client": + m = nw_config.maca(int(nw_config.MAC_L_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1)) + if nodetype == "server": + m = nw_config.maca(int(nw_config.MAC_R_START) + int(nw_config.maca(nw_config.ENI_MAC_STEP)) * (vpc - 1) + + (int(nw_config.maca(nw_config.ACL_TABLE_MAC_STEP)) * count)) return str(m).replace('-', ':') @@ -1041,24 +1091,73 @@ def build_node_vlan(index, nw_config, nodetype="client"): return vlan -def create_ip_list(nw_config): +def create_staticarp_ranges(nw_config, ip_list): + + staticarp_ranges = {'client': [], 'server': []} + + # client side + count = 0 + NETWORK_RANGE_ID = 0 + VLAN_ID = 1 + CLIENT_BASE_IP = int(nw_config.ipp("1.4.0.1")) + INCREMENT_ENI = int(nw_config.ipp("0.64.0.0")) + INCREMENT_NSG = int(nw_config.ipp("0.2.0.0")) + + for eni in range(0, nw_config.ENI_COUNT): + for eni_j in range(0, 10): + FIRST_IP = str(nw_config.ipp(CLIENT_BASE_IP + eni * INCREMENT_ENI + eni_j * INCREMENT_NSG)) + mac_client = build_node_macs(eni, eni_j+1, nw_config, 'privatelink', nodetype="staticarp_client") + payload = { + "firstIp": FIRST_IP, + "mac": mac_client, + "vlanId": VLAN_ID + 1000, + } + + staticarp_ranges['client'].append(payload) + count += 1 + NETWORK_RANGE_ID += 1 + # logger.info(f"Creating staticarp range for eni {eni} eni_j {eni_j} mac {mac_client} firstIp {FIRST_IP}") + VLAN_ID += 1 + + # server side + for eni in ip_list: + payload = { + "firstIp": eni['ip_client'], + "mac": eni['mac_client'], + "vlanId": eni['vlan_server'], + } + staticarp_ranges['server'].append(payload) + + return staticarp_ranges + + +def create_ip_list(nw_config, service_type): ip_list = [] + ENI_START = nw_config.ENI_START ENI_COUNT = nw_config.ENI_COUNT logger.info("Creating an ENI_COUNT = {} for l47 trafficgen".format(ENI_COUNT)) - for eni in range(nw_config.ENI_START, ENI_COUNT + 1): + for eni in range(ENI_START, ENI_COUNT + 1): ip_dict_temp = {} - ip_client = build_node_ips(0, eni, nw_config, nodetype="client") - mac_client = build_node_macs(0, eni, nw_config, nodetype="client") + + if service_type == 'privatelink': + dut_ip_client = build_node_ips(0, eni, nw_config, service_type, nodetype="dut_client") + dut_vlan_client = build_node_vlan(eni - 1, nw_config, nodetype="dut_client") + + ip_client = build_node_ips(0, eni, nw_config, service_type, nodetype="client") + mac_client = build_node_macs(0, eni, nw_config, service_type, nodetype="client") vlan_client = build_node_vlan(eni - 1, nw_config, nodetype="client") - ip_server = build_node_ips(0, eni, nw_config, nodetype="server") - mac_server = build_node_macs(0, eni, nw_config, nodetype="server") + ip_server = build_node_ips(0, eni, nw_config, service_type, nodetype="server") + mac_server = build_node_macs(0, eni, nw_config, service_type, nodetype="server") vlan_server = build_node_vlan(eni - 1, nw_config, nodetype="server") ip_dict_temp['eni'] = eni + if service_type == 'privatelink': + ip_dict_temp['dut_ip_client'] = dut_ip_client + ip_dict_temp['dut_vlan_client'] = dut_vlan_client ip_dict_temp['ip_client'] = ip_client ip_dict_temp['mac_client'] = mac_client ip_dict_temp['vlan_client'] = vlan_client @@ -1082,10 +1181,8 @@ def edit_l1_settings(api): for i in range(2): portl1_url = "ixload/test/activeTest/communityList/{}/network/portL1Settings".format(i) try: - # Code that may raise an exception - res = api.ixload_configure("patch", portl1_url, params) # noqa: F841 + api.ixload_configure("patch", portl1_url, params) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return @@ -1118,7 +1215,7 @@ def get_objectIDs(api, url): return objectIDs -def set_rangeList(api): +def set_rangeList(api, service_type): """ Adjust both rangeList, macRange, and vlanRange as needed """ @@ -1142,45 +1239,48 @@ def set_rangeList(api): for i, cid in enumerate(client_objectIDs): try: - # Code that may raise an exception - res1 = api.ixload_configure("patch", "{}/{}".format(clientList_url, cid), dict1) # noqa: F841 - res2 = api.ixload_configure("patch", "{}/{}".format(clientList_url, cid), dict2) # noqa: F841 - res3 = api.ixload_configure("patch", "{}/{}/vlanRange".format(clientList_url, cid), vlan_dict) # noqa: F841 + if service_type != 'privatelink': + api.ixload_configure("patch", "{}/{}".format(clientList_url, cid), dict1) + api.ixload_configure("patch", "{}/{}".format(clientList_url, cid), dict2) + api.ixload_configure("patch", "{}/{}/vlanRange".format(clientList_url, cid), vlan_dict) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") for i, sid in enumerate(server_objectIDs): try: - # Code that may raise an exception - res1 = api.ixload_configure("patch", "{}/{}/vlanRange".format(serverList_url, sid), vlan_dict) # noqa: F841 + api.ixload_configure("patch", "{}/{}/vlanRange".format(serverList_url, sid), vlan_dict) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return -def set_trafficMapProfile(api): +def set_trafficMapProfile(api, service_type): # Make Traffic Map Settings portMapPolicy_json = {'portMapPolicy': 'customMesh'} - destination_url = "ixload/test/activeTest/communityList/0/activityList/0/destinations/0" + + if service_type == 'vnet2vnet': + destination_url = "ixload/test/activeTest/communityList/0/activityList/0/destinations/0" + else: + destination_url = "ixload/test/activeTest/communityList/0/activityList/0/destinations/1" + try: - # Code that may raise an exception - res = api.ixload_configure("patch", destination_url, portMapPolicy_json) + api.ixload_configure("patch", destination_url, portMapPolicy_json) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") # meshType meshType_json = {'meshType': 'vlanRangePairs'} - submapsIpv4_url = "ixload/test/activeTest/communityList/0/activityList/0/destinations/0/customPortMap/submapsIPv4/0" + if service_type == 'vnet2vnet': + submapsIpv4_url = ("ixload/test/activeTest/communityList/0/activityList/0/destinations/0/" + "customPortMap/submapsIPv4/0") + else: + submapsIpv4_url = ("ixload/test/activeTest/communityList/0/activityList/0/destinations/1/customPortMap/" + "submapsIPv4/0") try: - # Code that may raise an exception - res = api.ixload_configure("patch", submapsIpv4_url, meshType_json) # noqa: F841 + api.ixload_configure("patch", submapsIpv4_url, meshType_json) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return @@ -1194,10 +1294,8 @@ def set_tcpCustom(api): param_json = {'maxPersistentRequests': 1} # response = requests.patch(url, json=param_json) try: - # Code that may raise an exception - res = api.ixload_configure("patch", tcp_agent_url, param_json) # noqa: F841 + api.ixload_configure("patch", tcp_agent_url, param_json) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return @@ -1205,10 +1303,10 @@ def set_tcpCustom(api): def set_timelineCustom(api, initial_cps_value): - activityList_url = "ixload/test/activeTest/communityList/0/activityList/0" # noqa: F841 + activityList_url = "ixload/test/activeTest/communityList/0/activityList/0" timelineObjectives_url = "ixload/test/activeTest/communityList/0/activityList/0/timeline" - activityList_json = { # noqa: F841 + activityList_json = { 'constraintType': 'ConnectionRateConstraint', 'constraintValue': initial_cps_value, 'enableConstraint': False, @@ -1220,11 +1318,194 @@ def set_timelineCustom(api, initial_cps_value): } try: - # Code that may raise an exception - res = api.ixload_configure("patch", timelineObjectives_url, timeline_json) # noqa: F841 + api.ixload_configure("patch", timelineObjectives_url, timeline_json) + api.ixload_configure("patch", activityList_url, activityList_json) + except Exception as e: + logger.info(f"An error occurred: {e}") + + return + + +def create_dut_config(api): + + param = { + "type": "VirtualDut" + } + + url = 'ixload/test/activeTest/dutList' + try: + api.ixload_configure("post", url, param) + except Exception as e: + logger.info(f"An error occurred: {e}") + + return + + +def add_dut_ranges(api, nw_config): + + data = {} + + # Here Create Ranges + url_networkRangeList = 'ixload/test/activeTest/dutList/0/dutConfig/networkRangeList' + for i in range((nw_config.ENI_COUNT * 10) - 1): + try: + api.ixload_configure("post", url_networkRangeList, data) + except Exception as e: + logger.info(f"An error occurred: {e}") + return None + + return + + +def patch_dut_config(api, nw_config, eni_per_dpu): + + param_comment = { + "comment": "DASH Private Link" + } + + url_attributes = 'ixload/test/activeTest/dutList/0' + + try: + api.ixload_configure("patch", url_attributes, param_comment) + except Exception as e: + logger.info(f"An error occurred: {e}") + return None + + count = 0 + NETWORK_RANGE_ID = 0 + VLAN_ID = 1 + BASE_IP = int(nw_config.ipp("1.4.0.1")) + INCREMENT_ENI = int(nw_config.ipp("0.64.0.0")) + INCREMENT_NSG = int(nw_config.ipp("0.2.0.0")) + + if eni_per_dpu == 64: + ipCount = 64 + else: + ipCount = nw_config.ENI_COUNT * 2 + + for i in range(0, nw_config.ENI_COUNT): + for j in range(0, 10): + FIRST_IP = str(nw_config.ipp(BASE_IP + i * INCREMENT_ENI + j * INCREMENT_NSG)) + payload = { + "firstIp": FIRST_IP, + "ipCount": ipCount, + "ipIncrStep": "0.0.0.2", + "networkMask": "255.192.0.0", + "vlanCount": 1, + 'vlanEnable': True, + "vlanId": VLAN_ID, + "vlanUniqueCount": 1 + } + url_networkRangeList_index = f'ixload/test/activeTest/dutList/0/dutConfig/networkRangeList/{count}' + + try: + api.ixload_configure("patch", url_networkRangeList_index, payload) + except Exception as e: + logger.info(f"An error occurred: {e}") + return None + + count += 1 + NETWORK_RANGE_ID += 1 + VLAN_ID += 1 + + return + + +def patch_destination_actionList(api): + + param_comment = { + "destination": "DUT1:80" + } + + url_actionList2 = '/ixload/test/activeTest/communityList/0/activityList/0/agent/actionList/2' + + try: + api.ixload_configure("patch", url_actionList2, param_comment) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") + return None + + return + + +def patch_communityList2(api, nw_config): + + url = "ixload/test/activeTest/communityList/1/network/stack/childrenList/5/childrenList/6/rangeList" + objectIDs = get_objectIDs(api, url) + + param_doubleIncrement = { + 'doubleIncrement': True, + } + + # secondCount is based on # VNET mapping per ENI 64K requirement. Adjust scale to set. + scale = 0.01 + first_count = nw_config.ACL_TABLE_COUNT * 2 + second_count = int((64000 * scale)/first_count) + param_doubleIncrement2 = { + 'firstIncrementBy': '::002:0', + 'firstCount': first_count, + 'gatewayAddress': '::0', + 'gatewayIncrement': '::0', + 'secondCount': second_count, + 'secondIncrementBy': '::2' + } + + for id in objectIDs: + url = f"ixload/test/activeTest/communityList/1/network/stack/childrenList/5/childrenList/6/rangeList/{id}" + try: + api.ixload_configure("patch", url, param_doubleIncrement) + api.ixload_configure("patch", url, param_doubleIncrement2) + except Exception as e: + logger.info(f"An error occurred: {e}") + return None + + return + + +def delete_staticarp_files(chassis_ip, tbinfo, ixos_version): + def delete_files(ssh_client, remote_path, logger=None): + """ + Delete a remote file using SFTP. If it doesn't exist, do nothing. + """ + sftp = None + try: + sftp = ssh_client.open_sftp() + try: + attrs = sftp.stat(remote_path) + if stat.S_ISDIR(attrs.st_mode): + msg = f"Refusing to delete directory: {remote_path}" + + logger.warning(msg) + return + except FileNotFoundError: + msg = f"File not found (nothing to delete): {remote_path}" + logger.info(msg) + return + + sftp.remove(remote_path) + msg = f"Deleted: {remote_path}" + logger.info(msg) + except Exception as e: + msg = f"Failed to delete {remote_path}: {e}" + logger.exception(msg) + finally: + if sftp: + sftp.close() + + # Connect to remote host + logger.info(f"Connecting to {chassis_ip}...") + ssh_client = set_chassis_connect(chassis_ip, tbinfo) + + card = 1 # Update this to use card number as an index + # client + client_path = os.path.join(f'/home/ixia_apps/ixos/{ixos_version}/nfs/rw/ports/{card}/1/ixtcp.arp') + delete_files(ssh_client, client_path, logger) + + # server + server_path = os.path.join(f'/home/ixia_apps/ixos/{ixos_version}/nfs/rw/ports/{card}/2/ixtcp.arp') + delete_files(ssh_client, server_path, logger) + + close_chassis_connect(ssh_client) return @@ -1238,33 +1519,145 @@ def set_userIPMappings(api): } try: - # Code that may raise an exception - res = api.ixload_configure("patch", url, param_userIpMapping) # noqa: F841 + api.ixload_configure("patch", url, param_userIpMapping) + except Exception as e: + logger.info(f"An error occurred: {e}") + return None + + return + + +def set_test_preferences(api): + + url = "ixload/preferences" + + param_allowRouteConflicts = { + 'allowRouteConflicts': True + } + + try: + api.ixload_configure("patch", url, param_allowRouteConflicts) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return None return -def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_filename, initial_cps_value): +def set_chassis_connect(chassis_ip, tbinfo): + + chassis_user_login = tbinfo.get('chassis_user_login') + chassis_user_passwd = tbinfo.get('chassis_user_passwd') + + # setting up ssh connection + try: + logger.info("Setting up an SSH client to connect to chassis") + ssh_client = paramiko.SSHClient() + logger.info("Loading host keys for SSH client") + ssh_client.load_system_host_keys() + logger.info(f"Connecting to {chassis_ip}...") + ssh_client.connect(hostname=f'{chassis_ip}', username=f'{chassis_user_login}', + password=f'{chassis_user_passwd}') + except Exception as e: + logger.error(f"Failed to connect to {chassis_ip}, make sure credentials are correct and network is " + f"reachable and chassis is part of known_hosts: {e}") + return None + + return ssh_client + + +def close_chassis_connect(ssh_client): + # Close the SSH connection + ssh_client.close() + + +def set_trafficgen_staticarps(staticarp_ranges, chassis_ip, tbinfo, + ixos_version, nw_config): + def build_arp_payload(ssh_client, staticarp_ranges, ENI_TOTAL, filename, is_server, remote_path): + lines = [] + if is_server: + count = 1 + for i in range(ENI_TOTAL): + mac_addr = staticarp_ranges['server'][i]['mac'] + ip_addr = staticarp_ranges['server'][i]['firstIp'] + vlan_id = staticarp_ranges['server'][i]['vlanId'] + lines.append( + f"RemoteMAC={mac_addr}, RemoteMACIncr=00:00:00:00:00:02, " # noqa: E231 + f"RemoteIPv4={ip_addr}, RemoteIPv4Incr=0.0.0.2, " + f"Count={count}, VlanID={vlan_id}\n") + else: + count = ENI_TOTAL * 2 + for i in range(ENI_TOTAL*10): + mac_addr = staticarp_ranges['client'][i]['mac'] + ip_addr = staticarp_ranges['client'][i]['firstIp'] + vlan_id = staticarp_ranges['client'][i]['vlanId'] + lines.append( + f"RemoteMAC={mac_addr}, RemoteMACIncr=00:00:00:00:00:02, " # noqa: E231 + f"RemoteIPv4={ip_addr}, RemoteIPv4Incr=0.0.0.2, " + f"Count={count}, VlanID={vlan_id}\n") + + fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w') as f: + f.writelines(lines) + + with SCPClient(ssh_client.get_transport()) as scp_client: + try: + scp_client.put(filename, remote_path) + except Exception as e: + logger.error(f"SCP failed arp payload upload to trafficgen server: {e}") + + os.remove(filename) + return + + ENI_TOTAL = nw_config.ENI_COUNT + card = 1 # Make this an index + + # Connect to remote host + logger.info(f"Connecting to {chassis_ip}...") + ssh_client = set_chassis_connect(chassis_ip, tbinfo) + + # Create client files + client_path = os.path.join(f'/home/ixia_apps/ixos/{ixos_version}/nfs/rw/ports/{card}/1/ixtcp.arp') + build_arp_payload( + ssh_client, staticarp_ranges, ENI_TOTAL, f"ixtcp_{card}.arp", False, client_path + ) + + # Create server files + server_path = os.path.join(f'/home/ixia_apps/ixos/{ixos_version}/nfs/rw/ports/{card}/2/ixtcp.arp') + build_arp_payload( + ssh_client, staticarp_ranges, ENI_TOTAL, f"ixtcp_{card}_1.arp", True, server_path + ) + + close_chassis_connect(ssh_client) + + logger.info("ARP files created successfully!") + + return + + +def l47_trafficgen_main(ports_list, tbinfo, connection_dict, nw_config, service_type, + test_type, test_filename, initial_cps_value): # Start Here ###### main_start_time = time.time() gw_ip = connection_dict['gw_ip'] port = connection_dict['port'] - chassis_ip = connection_dict['chassis_ip'] + chassis_ip = tbinfo.get('chassis_ip') + eni_per_dpu = int(tbinfo.get('eni_per_dpu', 32)) ixl_version = connection_dict['version'] + ixos_version = connection_dict['ixos_version'] api = snappi.api(location="{}:{}".format(gw_ip, port), ext="ixload", verify=False, version=ixl_version) config = api.config() - port_1 = config.ports.port(name="p1", location="{}/1/1".format(chassis_ip))[-1] # noqa: F841 - port_2 = config.ports.port(name="p2", location="{}/1/2".format(chassis_ip))[-1] # noqa: F841 + config.ports.port(name="p1", location="{}/1/1".format(chassis_ip)) + config.ports.port(name="p2", location="{}/1/2".format(chassis_ip)) # client/server IP ranges created here - ip_list = create_ip_list(nw_config) + ip_list = create_ip_list(nw_config, service_type) + if service_type == 'privatelink': + staticarp_ranges = create_staticarp_ranges(nw_config, ip_list) + set_trafficgen_staticarps(staticarp_ranges, chassis_ip, tbinfo, ixos_version, nw_config) logger.info("Setting devices") time_device_time = time.time() @@ -1319,12 +1712,20 @@ def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_ eth2.step = "00:00:00:00:00:02" # ip section - ip2 = eth2.ipv4_addresses.ipv4()[-1] - ip2.name = "{}.ipv4".format(eth2.name) - ip2.address = eni_info['ip_server'] - ip2.prefix = 10 - ip2.gateway = "0.0.0.0" - ip2.count = 1 + if service_type == 'privatelink': + ip2 = eth2.ipv6_addresses.ipv6()[-1] + ip2.name = "{}.ipv6".format(eth2.name) + ip2.address = eni_info['ip_server'] + ip2.prefix = 128 + ip2.gateway = "::0" + # ip2.count = 1 + else: + ip2 = eth2.ipv4_addresses.ipv4()[-1] + ip2.name = "{}.ipv4".format(eth2.name) + ip2.address = eni_info['ip_server'] + ip2.prefix = 10 + ip2.gateway = "0.0.0.0" + ip2.count = 1 # vlan section vlan2 = eth2.vlans.vlan()[-1] @@ -1484,7 +1885,7 @@ def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_ # Set config logger.info("Configuring custom settings") time_custom_time = time.time() - response = api.set_config(config) # noqa: F841 + api.set_config(config) port = connection_dict['port'] time_custom_finish = time.time() @@ -1495,9 +1896,23 @@ def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_ set_userIPMappings(api) logger.info("userIpMapping completed") + # Create DUTLIST + if service_type == 'privatelink': + logger.info("Configuring DUT list settings") + create_dut_config(api) + add_dut_ranges(api, nw_config) + patch_dut_config(api, nw_config, eni_per_dpu) + + # Patch Traffic between Network1 and DUT + patch_destination_actionList(api) + + # Patch Network 2 IPv6 + patch_communityList2(api, nw_config) + logger.info("Configuring custom port settings") time_assignPort_time = time.time() assignPorts(api, ports_list) + checkPorts(api, ports_list) time_assignPort_finish = time.time() logger.info("Custom port settings completed: {}".format(time_assignPort_finish - time_assignPort_time)) @@ -1510,14 +1925,14 @@ def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_ # Here adjust Double Increment and vlanRange unique number logger.info("Configuring rangeList settings for client and server") test_rangeList_time = time.time() - set_rangeList(api) + set_rangeList(api, service_type) test_rangeList_finish_time = time.time() logger.info("rangeList settings completed {}".format(test_rangeList_finish_time-test_rangeList_time)) # Adjust Traffic Profile logger.info("Custom trafficmaps") test_trafficmaps_time = time.time() - set_trafficMapProfile(api) + set_trafficMapProfile(api, service_type) test_trafficmaps_finish = time.time() logger.info("Finished traffic maps configuration {}".format(test_trafficmaps_finish - test_trafficmaps_time)) @@ -1534,11 +1949,10 @@ def l47_trafficgen_main(ports_list, connection_dict, nw_config, test_type, test_ test_timeline_finish = time.time() logger.info("Finished timeline configurations {}".format(test_timeline_finish - test_timeline_time)) - # save file - # logger.info("Saving Test File") - test_save_time = time.time() # noqa: F841 - test_save_finish_time = time.time() # noqa: F841 - # logger.info("Finished saving: {}".format(test_save_finish_time - test_save_time)) + # allow route conflicts + logger.info("Setting test preferences to allow route conflicts to enabled") + set_test_preferences(api) + main_finish_time = time.time() logger.info("Ixload configuration app finished in {}".format(main_finish_time - main_start_time)) @@ -1556,10 +1970,8 @@ def saveAs(api, test_filename): # response = requests.post(url, data=json.dumps(paramDict), headers=headers) try: - # Code that may raise an exception - res = api.ixload_configure("post", saveAs_operation, paramDict) # noqa: F841 + api.ixload_configure("post", saveAs_operation, paramDict) except Exception as e: - # Handle any exception logger.info(f"An error occurred: {e}") return diff --git a/tests/common/snappi_tests/snappi_fixtures.py b/tests/common/snappi_tests/snappi_fixtures.py index e6fb3e298ae..b68e51629d2 100755 --- a/tests/common/snappi_tests/snappi_fixtures.py +++ b/tests/common/snappi_tests/snappi_fixtures.py @@ -26,8 +26,9 @@ prefix_length, dut_ipv6_start, snappi_ipv6_start, v6_prefix_length, dut_ip_for_non_macsec_port from tests.common.macsec.macsec_config_helper import set_macsec_profile, enable_macsec_port, disable_macsec_port, \ delete_macsec_profile -from tests.common.snappi_tests.uhd.uhd_helpers import NetworkConfigSettings, create_front_panel_ports, \ - create_connections, create_uhdIp_list, create_arp_bypass, create_profiles +from tests.common.snappi_tests.uhd.uhd_helpers import (NetworkConfigSettings, create_front_panel_ports, + create_connections, create_connections_pl, create_uhdIp_list, + create_arp_bypass, create_arp_bypass_pl, create_profiles) logger = logging.getLogger(__name__) _next_system_id = 1 @@ -2167,10 +2168,14 @@ def read_links_from_csv(file_path): ethpass_ports = [row for row in csv_data if row['EthernetPass'] == 'True'] has_switchover = any(dpu.get('SwitchOverPort') == 'True' for dpu in dpu_ports) + service_type = tbinfo['service_type'] uhdConnect_ip = tbinfo['uhd_ip'] num_cps_cards = tbinfo['num_cps_cards'] num_tcpbg_cards = tbinfo['num_tcpbg_cards'] num_udpbg_cards = tbinfo['num_udpbg_cards'] + vxlan_port = tbinfo.get('vxlan_port', 0) + vxlan_src_port = tbinfo.get('vxlan_src_port', 0) + vxlan_endpoint_vni = tbinfo.get('vxlan_endpoint_vni', 1000) num_dpu_ports = len(dpu_ports) cards_dict = { @@ -2184,7 +2189,7 @@ def read_links_from_csv(file_path): 'switchover_port': has_switchover } - uhdSettings = NetworkConfigSettings() # noqa: F405 + uhdSettings = NetworkConfigSettings(vxlan_endpoint_vni) # noqa: F405 uhdSettings.set_mac_addresses(tbinfo['l47_tg_clientmac'], tbinfo['l47_tg_servermac'], tbinfo['dut_mac']) total_cards = num_cps_cards + num_tcpbg_cards + num_udpbg_cards subnet_mask = uhdSettings.subnet_mask @@ -2192,9 +2197,19 @@ def read_links_from_csv(file_path): logger.info(f"Configuring UHD connect for {uhdSettings.ENI_COUNT} ENIs") ip_list = create_uhdIp_list(subnet_mask, uhdSettings, cards_dict) # noqa: F405 fp_ports_list = create_front_panel_ports(int(total_cards * 2), uhdSettings, cards_dict) # noqa: F405 - arp_bypass_list = create_arp_bypass(fp_ports_list, ip_list, uhdSettings, cards_dict, subnet_mask) # noqa: F405 - connections_list = create_connections(fp_ports_list, ip_list, subnet_mask, uhdSettings, # noqa: F405 - cards_dict, arp_bypass_list) + + if service_type == 'vnet2vnet': + file_name = "tempUhdConfig_vnet2vnet.json" + arp_bypass_list = create_arp_bypass(fp_ports_list, ip_list, uhdSettings, cards_dict, + subnet_mask) + connections_list = create_connections(fp_ports_list, ip_list, subnet_mask, uhdSettings, # noqa: F405 + cards_dict, arp_bypass_list, vxlan_port, vxlan_src_port) + else: + # privatelink + file_name = "tempUhdConfig_pl.json" + arp_bypass_list = create_arp_bypass_pl(fp_ports_list, ip_list, uhdSettings, cards_dict, subnet_mask) + connections_list = create_connections_pl(fp_ports_list, ip_list, subnet_mask, uhdSettings, cards_dict, + arp_bypass_list, vxlan_port, vxlan_src_port) config = { "profiles": create_profiles(uhdSettings), # noqa: F405 @@ -2206,7 +2221,6 @@ def read_links_from_csv(file_path): 'Content-Type': 'application/json' } - file_name = "tempUhdConfig.json" file_location = os.getcwd() uhd_post_url = uhdSettings.uhd_post_url url = "https://{}/{}".format(uhdConnect_ip, uhd_post_url) # noqa: F841 @@ -2215,7 +2229,10 @@ def read_links_from_csv(file_path): logger.info(f"Pushing created UHD configuration file {file_name} to UHD Connect") uhdConf_cmd = ('curl -k -X POST -H \"Content-Type: application/json\" -d @\"{}/{}\" ' '{}').format(file_location, file_name, url) - subprocess.run(uhdConf_cmd, shell=True, capture_output=True, text=True) + try: + res = subprocess.run(uhdConf_cmd, shell=True, capture_output=True, text=True) # noqa: F841 + except Exception as e: + logger.error(f"UHD config upload failed: {e}") if not save_uhd_config: logger.info("Removing UHD config file") diff --git a/tests/common/snappi_tests/uhd/uhd_helpers.py b/tests/common/snappi_tests/uhd/uhd_helpers.py index 8f9746c5689..ad74490062a 100644 --- a/tests/common/snappi_tests/uhd/uhd_helpers.py +++ b/tests/common/snappi_tests/uhd/uhd_helpers.py @@ -6,7 +6,7 @@ class NetworkConfigSettings: - def __init__(self): + def __init__(self, vxlan_endpoint_vni=1000): self.ipp = ipaddress.ip_address self.maca = macaddress.MAC @@ -15,8 +15,10 @@ def __init__(self): self.ENI_COUNT = 256 self.ENI_MAC_STEP = '00:00:00:18:00:00' self.ENI_STEP = 1 - self.ENI_L2R_STEP = 1000 + self.ENI_L2R_STEP = vxlan_endpoint_vni + self.ENI_PER_PORT = 64 + self.VTEP_IP = self.ipp("221.0.0.1") self.PAL = self.ipp("221.1.0.1") self.PAR = self.ipp("221.2.0.1") self.STATIC = self.ipp("221.0.0.1") @@ -26,6 +28,7 @@ def __init__(self): self.ACL_TABLE_MAC_STEP = '00:00:00:02:00:00' self.ACL_POLICY_MAC_STEP = '00:00:00:00:00:32' + self.ENI_VNI_NVGRE_START = 1000 self.ACL_RULES_NSG = 1000 self.ACL_TABLE_COUNT = 5 @@ -41,6 +44,8 @@ def __init__(self): self.IP_L_START = self.ipp('1.1.0.1') self.IP_R_START = self.ipp('1.4.0.1') + self.IPv6_R_START = self.ipp('2603:100:3E8::104:1') + self.IPv6_Range_Increment = self.ipp('0:0:1::40:0') self.MAC_L_START = self.maca('00:1A:C5:00:00:01') self.MAC_R_START = self.maca('00:1B:6E:00:00:01') @@ -57,6 +62,19 @@ def __init__(self): self.first_staticArpMac = '' self.dut_mac = '' + # PrivateLink + self.NVGRE_COUNT = 32 + self.NVGRE_SUBMASK = 48 + self.NVGRE_VSID = 100 + self.NVGRE_IP = self.ipp('2603:100:3e8::0:0') + self.NVGRE_IP_STEP = self.ipp(ipaddress.ip_address('0:0:1::0:0')) + self.PAL_PL = self.ipp("221.2.0.0") + self.PAR_PL = self.ipp("221.1.0.0") + + def dec2hex(self, dec): + hex_str = hex(dec)[2:] + return hex_str + def set_mac_addresses(self, clientmac, servermac, dutmac): self.l47_tg_clientmac = str(self.maca(clientmac)).replace('-', ':') @@ -72,9 +90,9 @@ def set_mac_addresses(self, clientmac, servermac, dutmac): def build_node_ips(count, vpc, config, nodetype="client"): - if nodetype in "client": + if nodetype == "client": ip = config.ipp(int(config.IP_R_START) + (config.IP_STEP_NSG * count) + int(config.IP_STEP_ENI) * (vpc - 1)) - if nodetype in "server": + if nodetype == "server": ip = config.ipp(int(config.IP_L_START) + int(config.IP_STEP_ENI) * (vpc - 1)) return str(ip) @@ -279,8 +297,6 @@ def create_front_panel_ports(count, config, cards_dict): data_index += 1 # l47 Front Panel DPU - # TODO add num_dpuPorts then build this part - dpu_ports_length = len(cards_dict['dpu_ports']) dpu_port = 0 switchover_port = 0 @@ -319,6 +335,208 @@ def create_front_panel_ports(count, config, cards_dict): return fp_list +def create_arp_bypass_pl(fp_ports_list, ip_list, config, cards_dict, subnet_mask): + + connections_list = [] + ethpass_ports = cards_dict['ethpass_ports'] + # num_cps_cards = cards_dict['num_cps_cards'] + # first_cps_card, first_tcpbg_card = set_first_stateful_cards(cards_dict) + + # eth_bypass set here + connections_list.append(_get_eth_bypass_dict(config, ethpass_ports)) + + return connections_list + + +def create_connections_pl(fp_ports_list, ip_list, subnet_mask, config, cards_dict, arp_bypass_list, + vxlan_port, vxlan_src_port): + + connections_list = arp_bypass_list + num_cps_cards = cards_dict['num_cps_cards'] + first_cps_card, first_tcpbg_card = set_first_stateful_cards(cards_dict) + + ip6_step = int(config.NVGRE_IP_STEP) + nvgre_count = config.NVGRE_COUNT + nvgre_eni = config.ENI_START + vtep_ip = config.VTEP_IP + vtep_ip_tmp = 0 + underlay_ip = config.PAL_PL + underlay_ip_tmp = 0 + vlan_endpoint_ip = config.PAR_PL + vlanEP_ip_tmp = 0 + client_vlan_start = config.ENI_L2R_STEP + 1 + client_vlan_tmp = 0 + # vxlan_vni_start = config.ENI_L2R_STEP * 2 + vxlan_vni_start = config.ENI_L2R_STEP + vxlan_vni_tmp = 0 + nvgre_ip = "2603:100:%s:0::0" % (config.dec2hex(config.ENI_VNI_NVGRE_START)) + nvgre_ip_tmp = 0 + + for port in range(num_cps_cards): + + server_dict_temp = { + 'name': 'l47 Server {}'.format(port+1), + 'functions': [], + 'endpoints': [] + } + + client_dict_temp = { + 'name': 'l47 Client {}'.format(port+1), + 'functions': [], + 'endpoints': [] + } + + # client_vlan = build_node_vlan(eni, config, nodetype="client") + # server_vlan = build_node_vlan(eni, config, nodetype="server") + + # client_card, test_role = find_card_slot(first_cps_card, first_tcpbg_card, server_vlan) + # client_cps_port = find_port(num_cps_cards, first_cps_card, first_tcpbg_card, server_vlan, test_role) + # server_card, test_role = find_card_slot(first_cps_card, first_tcpbg_card, server_vlan) + + # Server side + """ + if server_vlan == 256: + overlay_ip_addr = 0 + else: + overlay_ip_addr = eni+1 + """ + # overlay_ip_addr = eni + + # if server_vlan <= 128: + # lb_ip = 1 + # else: + # lb_ip = 2 + + # vni_index = 1000 + + nvgre_ip_tmp = str(ipaddress.ip_address(nvgre_ip) + (ip6_step * (port * nvgre_count))) + vtep_ip_tmp = vtep_ip + (1 * port) + underlay_ip_tmp = underlay_ip + (config.NVGRE_COUNT * port) + server_conn_tmp = { + "choice": "connect_vlan_nvgre", + "connect_vlan_nvgre": { + "vlan_endpoint_settings": { + "outgoing_nvgre_header": { + "src_mac": {"choice": "mac", "mac": "{}".format(config.l47_tg_servermac)}, + "dst_mac": {"choice": "mac", "mac": "{}".format(config.dut_mac)}, + "src_ip": {"choice": "ipv4_range", "ipv4_range": {'start': "{}".format(underlay_ip_tmp), + 'count': nvgre_count, 'step': '0.0.0.1'}}, + "dst_ip": {"choice": "ipv4", "ipv4": "{}".format(vtep_ip_tmp)}, + } + }, + "nvgre_endpoint_settings": { + "vsid": {"choice": "vsid", "vsid": config.NVGRE_VSID}, + "protocols": {"accept": ["tcp"]}, "routing_method": "ip_routing", # noqa: E128 + "ip_routing": {"destination_ips": {"choice": "ipv6_range", + "ipv6_range": {'start': "{}".format(ipaddress.ip_address(nvgre_ip_tmp)), # noqa: E128 + 'subnet_bits': config.NVGRE_SUBMASK, 'count': nvgre_count, + 'step': "{}".format(str(ipaddress.ip_address(ip6_step)))}}} + } + } + } + server_dict_temp['functions'].append(server_conn_tmp) + nvgre_eni_temp = nvgre_eni + (nvgre_count * port) + server_dict_temp['endpoints'].append( + {"choice": "front_panel", "front_panel": { + "port_name": "l47_port_{}s".format(port + 1), # noqa: E122 + "vlan": {"choice": "vlan_range", "vlan_range": {"start": nvgre_eni_temp, "count": 32, "step": 1}}}, + "tags": ["vlan"]}, + ) + server_dict_temp['endpoints'].append( + {"choice": "front_panel", "front_panel": {"port_name": "l47_dpuPort_{}".format(1)}, + "tags": ["nvgre"]} + ) + + # Client Side + client_start = config.IP_L_START + ip_tmp = client_start + (int(ipaddress.ip_address('8.0.0.0')) * port) + vlanEP_ip_tmp = vlan_endpoint_ip + (nvgre_count * port) + vxlan_vni_tmp = vxlan_vni_start + (nvgre_count * port) + + client_conn_tmp = { + "choice": "connect_vlan_vxlan", + "connect_vlan_vxlan": { + "vlan_endpoint_settings": { + "outgoing_vxlan_header": { + "src_mac": {"choice": "mac", "mac": f"{config.l47_tg_clientmac}"}, + "dst_mac": {"choice": "mac", "mac": f"{config.dut_mac}"}, + "src_ip": { + "choice": "ipv4_range", + "ipv4_range": { + "start": f"{vlanEP_ip_tmp}", + "count": nvgre_count, + "step": "0.0.0.1", + }, + }, + "dst_ip": {"choice": "ipv4", "ipv4": f"{vtep_ip_tmp}"}, + } + }, + "vxlan_endpoint_settings": { + "vni": { + "choice": "vni_range", + "vni_range": { + "start": vxlan_vni_tmp, + "count": nvgre_count, + "step": 1, + }, + }, + "protocols": {"accept": ["tcp"]}, + "routing_method": "ip_routing", + "ip_routing": { + "destination_ips": { + "choice": "ipv4_range", + "ipv4_range": { + "start": f"{ip_tmp}", + "count": config.NVGRE_COUNT, + "step": "0.64.0.0", + }, + } + }, + }, + }, + } + + if not (vxlan_port == 0 and vxlan_src_port == 0): + vxlan_settings = client_conn_tmp["connect_vlan_vxlan"]["vxlan_endpoint_settings"] + vxlan_settings["udp_src_port"] = vxlan_port + vxlan_settings["udp_dst_port"] = vxlan_src_port + + client_vlan_tmp = client_vlan_start + (nvgre_count * port) + client_dict_temp['functions'].append(client_conn_tmp) + client_dict_temp['endpoints'].append( + {"choice": "front_panel", "front_panel": { + "port_name": "l47_port_{}c".format(port+1), + "vlan": {"choice": "vlan_range", "vlan_range": {'start': client_vlan_tmp, 'count': nvgre_count, + 'step': 1}}}, "tags": ["vlan"]}, + ) + + client_dict_temp['endpoints'].append( + {"choice": "front_panel", "front_panel": {"port_name": "l47_dpuPort_{}".format(1)}, + "tags": ["vxlan"]} + ) + + # Add server and client settings to connections_list + connections_list.append(server_dict_temp) + connections_list.append(client_dict_temp) + + return connections_list + + +def _get_eth_bypass_dict(config, ethpass_ports): + eth_bypass_dict = { + 'name': "Eth Bypass", + 'functions': [{"choice": "connect_ethernet", "connect_ethernet": {}}], + 'endpoints': [] + } + for port in ethpass_ports: + eth_bypass_dict['endpoints'].append( + {'choice': 'front_panel', 'front_panel': {'port_name': 'l47_port_11{}'.format(port['FrontPanel']), + 'vlan': {'choice': 'non_vlan'}}} + ) + + return eth_bypass_dict + + def create_arp_bypass(fp_ports_list, ip_list, config, cards_dict, subnet_mask): connections_list = [] @@ -334,6 +552,8 @@ def create_arp_bypass(fp_ports_list, ip_list, config, cards_dict, subnet_mask): else: first_tcpbg_card = 0 """ + + """ eth_bypass_dict = { 'name': "Eth Bypass", 'functions': [{"choice": "connect_ethernet", "connect_ethernet": {}}], @@ -347,6 +567,9 @@ def create_arp_bypass(fp_ports_list, ip_list, config, cards_dict, subnet_mask): {'port_name': 'l47_port_11{}'.format(port['FrontPanel']), 'vlan': {'choice': 'non_vlan'}}} ) connections_list.append(eth_bypass_dict) + """ + # eth_bypass set here + connections_list.append(_get_eth_bypass_dict(config, ethpass_ports)) for eni, ip in enumerate(ip_list): @@ -360,7 +583,7 @@ def create_arp_bypass(fp_ports_list, ip_list, config, cards_dict, subnet_mask): server_vlan = build_node_vlan(eni, config, nodetype="server") client_card, test_role = find_card_slot(config, cards_dict, first_cps_card, first_tcpbg_card, server_vlan) - client_port = find_port(num_cps_cards, first_cps_card, first_tcpbg_card, server_vlan, test_role) # noqa: F841 + find_port(num_cps_cards, first_cps_card, first_tcpbg_card, server_vlan, test_role) # server_card, test_role = find_card_slot(first_cps_card, first_tcpbg_card, server_vlan) if cards_dict['num_tcpbg_cards'] > 0: @@ -383,18 +606,13 @@ def create_arp_bypass(fp_ports_list, ip_list, config, cards_dict, subnet_mask): return connections_list -def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, arp_bypass_list): +def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, arp_bypass_list, + vxlan_port=0, vxlan_src_port=0): connections_list = arp_bypass_list first_cps_card, first_tcpbg_card = set_first_stateful_cards(cards_dict) - """ - if cards_dict['num_cps_cards'] > 0: - first_cps_card = 1 - first_tcpbg_card = cards_dict['num_cps_cards'] + 1 - """ - # TODO loopback IP need updated for multiple DPUs for example: 'dst_ip': {'choice': 'ipv4', 'ipv4': '221.0.0.1'} for eni, ip in enumerate(ip_list): server_dict_temp = { @@ -416,7 +634,6 @@ def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, # client_cps_port = find_port(num_cps_cards, first_cps_card, first_tcpbg_card, server_vlan, test_role) server_card, test_role = find_card_slot(config, cards_dict, first_cps_card, first_tcpbg_card, server_vlan) - # TODO needed when there are multiple DPU Ports # dpu_port = 1 if server_vlan <= 128 else 2 # dpu_port = 1 @@ -438,12 +655,7 @@ def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, client_role, server_role = find_testrole(test_role, server_vlan) - # TODO VNIs need to be +1000 for production - production = True # turn ON for now - if production is True: - vni_index = 1000 - else: - vni_index = 0 + vni_index = 1000 server_conn_tmp = {"choice": "connect_vlan_vxlan", "connect_vlan_vxlan": { "vlan_endpoint_settings": { @@ -459,6 +671,12 @@ def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, "ip_routing": {"destination_ips": {"choice": "ipv4", "ipv4": "{}".format(ip_list[eni]['ip_server'])}}} # noqa: E128 }} + + if not (vxlan_port == 0 and vxlan_src_port == 0): + server_vxlan_settings = server_conn_tmp["connect_vlan_vxlan"]["vxlan_endpoint_settings"] + server_vxlan_settings["udp_src_port"] = vxlan_port + server_vxlan_settings["udp_dst_port"] = vxlan_src_port + server_dict_temp['functions'].append(server_conn_tmp) server_dict_temp['endpoints'].append( {"choice": "front_panel", "front_panel": { @@ -471,7 +689,6 @@ def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, ) # Client Side - # TODO vni_index client_conn_tmp = {"choice": "connect_vlan_vxlan", "connect_vlan_vxlan": { "vlan_endpoint_settings": { "outgoing_vxlan_header": { @@ -490,6 +707,11 @@ def create_connections(fp_ports_list, ip_list, subnet_mask, config, cards_dict, }}}} }} + if not (vxlan_port == 0 and vxlan_src_port == 0): + client_vxlan_settings = client_conn_tmp["connect_vlan_vxlan"]["vxlan_endpoint_settings"] + client_vxlan_settings["udp_src_port"] = vxlan_port + client_vxlan_settings["udp_dst_port"] = vxlan_src_port + client_dict_temp['functions'].append(client_conn_tmp) client_dict_temp['endpoints'].append( {"choice": "front_panel", "front_panel": { diff --git a/tests/common/unit_tests/fixtures/unit_test_dash_utils.py b/tests/common/unit_tests/fixtures/unit_test_dash_utils.py new file mode 100644 index 00000000000..58a6bea715c --- /dev/null +++ b/tests/common/unit_tests/fixtures/unit_test_dash_utils.py @@ -0,0 +1,557 @@ +"""Unit tests for the DASH config ordering helpers in +``tests/common/dash_utils.py``. + +These tests load the target module in isolation using ``importlib`` so we +don't drag in the wider sonic-mgmt fixture stack. The module also does +``from constants import TEMPLATE_DIR`` at import time; we stub that out with +a fake ``constants`` module before loading so no DASH-test path needs to be +on ``sys.path``. + +Run with:: + + python3 -m pytest --noconftest tests/common/unit_tests/fixtures/unit_test_dash_utils.py -v +""" + +import importlib.util +import logging +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + + +MODULE_PATH = (Path(__file__).resolve().parents[3] + / "common" / "dash_utils.py") + + +def _install_stub_modules(): + """Install minimal stubs for modules ``dash_utils`` imports at module load + time so the unit tests don't depend on the sonic-mgmt test path layout.""" + if "constants" not in sys.modules: + constants_stub = types.ModuleType("constants") + constants_stub.TEMPLATE_DIR = "/tmp/_unit_test_template_dir" + sys.modules["constants"] = constants_stub + + # ``dash_utils`` also imports ``ptf.packet``, ``ptf.testutils``, ``pytest``, + # and ``jinja2``. ptf isn't installed in lightweight unit-test environments, + # so stub it; the other three should be present. + if "ptf" not in sys.modules: + ptf_stub = types.ModuleType("ptf") + ptf_packet_stub = types.ModuleType("ptf.packet") + ptf_testutils_stub = types.ModuleType("ptf.testutils") + sys.modules["ptf"] = ptf_stub + sys.modules["ptf.packet"] = ptf_packet_stub + sys.modules["ptf.testutils"] = ptf_testutils_stub + + +def _load_target_module(): + _install_stub_modules() + spec = importlib.util.spec_from_file_location( + "unit_target_dash_utils", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + # Register in sys.modules so test helpers can also resolve it by name + # (and so any internal `importlib.import_module` lookups work). + sys.modules["unit_target_dash_utils"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def dash_utils(): + return _load_target_module() + + +@pytest.fixture(autouse=True) +def _bypass_repo_log_format(monkeypatch): + """The repo's ``tests/pytest.ini`` configures ``log_format`` with a + custom ``%(funcNamewithModule)s`` field that is injected by the + ``log_section_start`` pytest plugin. Under ``--noconftest`` that plugin + isn't loaded, so any log record emitted during a test crashes pytest's + auto-attached ``LogCaptureHandler`` with ``KeyError: 'funcNamewithModule'``. + + Replace pytest's percent-style formatter with a plain ``%(message)s`` + formatter for the duration of every test so emitted warnings don't blow + up unrelated tests.""" + try: + import _pytest.logging as _pylog + except ImportError: # pragma: no cover - pytest internals always present + return + plain = logging.Formatter("%(message)s") + monkeypatch.setattr( + _pylog.PercentStyleMultiline, "format", + lambda self, record: plain.format(record), + ) + + +# --------------------------------------------------------------------------- # +# dash_table_name +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("key,expected", [ + ("DASH_APPLIANCE_TABLE", "APPLIANCE"), + ("DASH_APPLIANCE_TABLE:100", "APPLIANCE"), + ("DASH_VNET_TABLE:Vnet1", "VNET"), + ("DASH_VNET_MAPPING_TABLE:Vnet1:10.2.0.100", "VNET_MAPPING"), + ("DASH_ENI_TABLE:497f23d7-f0ac-4c99-a98f-59b470e8c7bd", "ENI"), + ("DASH_ENI_ROUTE_TABLE:eni-id", "ENI_ROUTE"), + ("DASH_ROUTING_TYPE_TABLE:privatelink", "ROUTING_TYPE"), + ("DASH_OUTBOUND_PORT_MAP_TABLE:portmap_1", "OUTBOUND_PORT_MAP"), + ("DASH_OUTBOUND_PORT_MAP_RANGE_TABLE:portmap_1:8001-9000", + "OUTBOUND_PORT_MAP_RANGE"), + ("DASH_ROUTE_RULE_TABLE:eni:100:1.2.3.4/32", "ROUTE_RULE"), +]) +def test_dash_table_name_parses_real_keys(dash_utils, key, expected): + assert dash_utils.dash_table_name(key) == expected + + +@pytest.mark.parametrize("bad_key", [ + "", + "FOO", + "VNET_TABLE:x", # missing DASH_ prefix + "DASH_VNET:foo", # missing _TABLE suffix + "dash_vnet_table:foo", # wrong case + "PREFIX_DASH_VNET_TABLE:foo", # DASH_ not at start +]) +def test_dash_table_name_rejects_non_dash_keys(dash_utils, bad_key): + with pytest.raises(ValueError): + dash_utils.dash_table_name(bad_key) + + +# --------------------------------------------------------------------------- # +# bucket_dash_configs +# --------------------------------------------------------------------------- # + +# --------------------------------------------------------------------------- # +# Canonical phase assignment for every known DASH table. Pinning these in a +# data-driven test makes accidental registry edits visible. +# --------------------------------------------------------------------------- # + +_EXPECTED_TABLE_PHASES = { + "APPLIANCE": "GROUP_1", + "ROUTING_TYPE": "GROUP_2", + "METER_POLICY": "GROUP_2", + "OUTBOUND_PORT_MAP": "GROUP_2", + "VNET": "GROUP_2", + "METER_RULE": "GROUP_3", + "TUNNEL": "GROUP_4", + "OUTBOUND_PORT_MAP_RANGE": "GROUP_4", + "ENI": "GROUP_4", + "ROUTE_GROUP": "GROUP_4", + "ROUTE_RULE": "GROUP_5", + "ROUTE": "GROUP_5", + "VNET_MAPPING": "GROUP_5", + "ENI_ROUTE": "GROUP_6", +} + + +def test_registry_contains_all_known_tables_with_expected_phases(dash_utils): + actual = {tbl: phase.name + for tbl, phase in dash_utils.DASH_TABLE_PHASE.items()} + assert actual == _EXPECTED_TABLE_PHASES + + +@pytest.mark.parametrize("table,phase_name", sorted(_EXPECTED_TABLE_PHASES.items())) +def test_each_table_buckets_into_its_canonical_phase(dash_utils, table, phase_name): + cfg = {"DASH_{}_TABLE:k".format(table): {"v": 1}} + buckets = dash_utils.bucket_dash_configs(cfg) + assert len(buckets) == 1 + phase, _ = buckets[0] + assert phase.name == phase_name, ( + "{} expected in phase {} but landed in {}".format( + table, phase_name, phase.name)) + + +def test_bucket_groups_by_phase_and_sorts_ascending(dash_utils): + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + vnet = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + eni_route = {"DASH_ENI_ROUTE_TABLE:eni-id": {"group_id": "rg1"}} + + # Pass them out of order to confirm the helper sorts. + buckets = dash_utils.bucket_dash_configs(eni_route, vnet, appliance, eni) + + phases = [p for p, _ in buckets] + assert phases == sorted(phases) + phase_names = [p.name for p in phases] + # APPLIANCE first (GROUP_1); VNET in GROUP_2; ENI in GROUP_4; ENI_ROUTE in GROUP_6. + assert phase_names == ["GROUP_1", "GROUP_2", "GROUP_4", "GROUP_6"] + + +def test_bucket_merges_same_phase_dicts_into_one_batch(dash_utils): + # TUNNEL, OUTBOUND_PORT_MAP_RANGE, ENI, ROUTE_GROUP all live in GROUP_4. + tunnel = {"DASH_TUNNEL_TABLE:T1": {"vni": 100}} + port_map_range = {"DASH_OUTBOUND_PORT_MAP_RANGE_TABLE:pm1:8001-9000": {}} + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + route_group = {"DASH_ROUTE_GROUP_TABLE:rg1": {"guid": "g"}} + + buckets = dash_utils.bucket_dash_configs( + tunnel, port_map_range, eni, route_group) + assert len(buckets) == 1 + phase, merged = buckets[0] + assert phase == dash_utils.DashPhase.GROUP_4 + assert set(merged) == { + "DASH_TUNNEL_TABLE:T1", + "DASH_OUTBOUND_PORT_MAP_RANGE_TABLE:pm1:8001-9000", + "DASH_ENI_TABLE:eni-id", + "DASH_ROUTE_GROUP_TABLE:rg1", + } + + +def test_bucket_empty_input_returns_empty_list(dash_utils): + assert dash_utils.bucket_dash_configs() == [] + assert dash_utils.bucket_dash_configs({}, {}) == [] + + +def test_bucket_warns_on_conflicting_duplicate_key(dash_utils, caplog): + a = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + b = {"DASH_VNET_TABLE:Vnet1": {"vni": 9999}} # same key, different value + with caplog.at_level(logging.WARNING, logger=dash_utils.logger.name): + buckets = dash_utils.bucket_dash_configs(a, b) + # Later value wins (matches {**a, **b} semantics). + _, merged = buckets[0] + assert merged["DASH_VNET_TABLE:Vnet1"] == {"vni": 9999} + assert any("Duplicate DASH key" in r.message for r in caplog.records) + + +def test_bucket_does_not_warn_on_identical_duplicate(dash_utils, caplog): + a = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + b = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + with caplog.at_level(logging.WARNING, logger=dash_utils.logger.name): + dash_utils.bucket_dash_configs(a, b) + assert not any("Duplicate DASH key" in r.message for r in caplog.records) + + +def test_bucket_unknown_table_falls_back_to_default_phase_with_warning( + dash_utils, caplog): + cfg = {"DASH_FUTURE_NEW_TABLE:x": {"foo": "bar"}} + with caplog.at_level(logging.WARNING, logger=dash_utils.logger.name): + buckets = dash_utils.bucket_dash_configs(cfg) + assert len(buckets) == 1 + phase, _ = buckets[0] + assert phase == dash_utils.DEFAULT_DASH_PHASE + assert any("Unknown DASH table" in r.message for r in caplog.records) + + +def test_bucket_rejects_non_dash_key(dash_utils): + with pytest.raises(ValueError): + dash_utils.bucket_dash_configs({"not_a_dash_key": {}}) + + +# --------------------------------------------------------------------------- # +# apply_dash_configs +# --------------------------------------------------------------------------- # + +def _fake_apply_fn(): + """Return a MagicMock that records each apply call's keys + set_db flag.""" + return MagicMock() + + +def _apply_call_table_summary(dash_utils, call): + """Pull out (set_db, sorted list of DASH table names) from a fake call.""" + args = call.args + # apply_fn(localhost, duthost, ptfhost, messages, dpu_index, set_db=..., ...) + messages = args[3] + set_db = call.kwargs.get("set_db", True) + tables = sorted({dash_utils.dash_table_name(k) for k in messages}) + return set_db, tables + + +def test_apply_calls_apply_fn_in_phase_order_for_set(dash_utils): + fake = _fake_apply_fn() + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + vnet = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + eni_route = {"DASH_ENI_ROUTE_TABLE:eni-id": {"group_id": "rg1"}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, + eni_route, vnet, appliance, eni, # out of order + apply_fn=fake, + ) + + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + # APPLIANCE (GROUP_1), VNET (GROUP_2), ENI (GROUP_4), ENI_ROUTE (GROUP_6). + assert summaries == [ + (True, ["APPLIANCE"]), + (True, ["VNET"]), + (True, ["ENI"]), + (True, ["ENI_ROUTE"]), + ] + + +def test_apply_reverses_order_for_set_db_false(dash_utils): + fake = _fake_apply_fn() + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + eni_route = {"DASH_ENI_ROUTE_TABLE:eni-id": {"group_id": "rg1"}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, + appliance, eni, eni_route, + set_db=False, + apply_fn=fake, + ) + + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + # On delete we tear down dependents first. + assert summaries == [ + (False, ["ENI_ROUTE"]), + (False, ["ENI"]), + (False, ["APPLIANCE"]), + ] + + +def test_apply_merges_same_phase_into_single_call(dash_utils): + fake = _fake_apply_fn() + vnet = {"DASH_VNET_TABLE:Vnet1": {"vni": 1000}} + meter_policy = {"DASH_METER_POLICY_TABLE:MP": {"ip_version": "v4"}} + port_map = {"DASH_OUTBOUND_PORT_MAP_TABLE:pm1": {}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, vnet, meter_policy, port_map, apply_fn=fake, + ) + # All three are in GROUP_2, so a single apply call. + assert fake.call_count == 1 + _, tables = _apply_call_table_summary(dash_utils, fake.call_args_list[0]) + assert tables == ["METER_POLICY", "OUTBOUND_PORT_MAP", "VNET"] + + +def test_apply_meter_policy_before_meter_rule(dash_utils): + """METER_RULE references METER_POLICY by name; METER_POLICY lands in + GROUP_2 and METER_RULE in GROUP_3 (between GROUP_2 and GROUP_4) so + meter rules are programmed after their parent policy but before any + ENI binds to that policy.""" + fake = _fake_apply_fn() + policy = {"DASH_METER_POLICY_TABLE:MP": {"ip_version": "v4"}} + rule = {"DASH_METER_RULE_TABLE:MP:1": {"priority": 0}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, rule, policy, apply_fn=fake) + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + assert summaries == [(True, ["METER_POLICY"]), (True, ["METER_RULE"])] + + +def test_apply_meter_rule_before_eni(dash_utils): + """METER_RULE must land before ENI so meter rules are present before + any ENI binds to their parent METER_POLICY.""" + fake = _fake_apply_fn() + rule = {"DASH_METER_RULE_TABLE:MP:1": {"priority": 0}} + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, eni, rule, apply_fn=fake) + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + assert summaries == [(True, ["METER_RULE"]), (True, ["ENI"])] + + +def test_apply_port_map_before_port_map_range(dash_utils): + """OUTBOUND_PORT_MAP_RANGE references OUTBOUND_PORT_MAP by name; + OUTBOUND_PORT_MAP lands in GROUP_2 and the RANGE in GROUP_4.""" + fake = _fake_apply_fn() + port_map = {"DASH_OUTBOUND_PORT_MAP_TABLE:pm1": {}} + port_map_range = {"DASH_OUTBOUND_PORT_MAP_RANGE_TABLE:pm1:8001-9000": {}} + + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, port_map_range, port_map, apply_fn=fake) + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + assert summaries == [ + (True, ["OUTBOUND_PORT_MAP"]), + (True, ["OUTBOUND_PORT_MAP_RANGE"]), + ] + + +def test_apply_eni_before_route_rule(dash_utils): + """Under the canonical phase ordering ENI lands in GROUP_4 and + ROUTE_RULE in GROUP_5, so ROUTE_RULE is applied after ENI exists.""" + fake = _fake_apply_fn() + eni = {"DASH_ENI_TABLE:eni-id": {"vnet": "Vnet1"}} + rule = {"DASH_ROUTE_RULE_TABLE:eni-id:100:1.2.3.4/32": {"priority": 0}} + + dash_utils.apply_dash_configs("lh", "dh", "ph", 0, eni, rule, apply_fn=fake) + summaries = [_apply_call_table_summary(dash_utils, c) + for c in fake.call_args_list] + assert summaries == [(True, ["ENI"]), (True, ["ROUTE_RULE"])] + + +def test_apply_with_no_configs_is_noop(dash_utils): + fake = _fake_apply_fn() + dash_utils.apply_dash_configs("lh", "dh", "ph", 0, apply_fn=fake) + dash_utils.apply_dash_configs("lh", "dh", "ph", 0, {}, {}, apply_fn=fake) + fake.assert_not_called() + + +def test_apply_skips_falsy_input_dicts(dash_utils): + """Callers use patterns like ``*(extra if cond else [])`` which may yield + nothing; we should silently ignore empty dicts.""" + fake = _fake_apply_fn() + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, appliance, {}, apply_fn=fake, + ) + assert fake.call_count == 1 + + +def test_apply_forwards_dpu_index_and_wait_kwargs(dash_utils): + fake = _fake_apply_fn() + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 7, appliance, + wait_after_apply=12, max_updates_in_single_cmd=64, + apply_fn=fake, + ) + call = fake.call_args_list[0] + # Positional args: localhost, duthost, ptfhost, messages, dpu_index + assert call.args[:3] == ("lh", "dh", "ph") + assert call.args[4] == 7 + assert call.kwargs["wait_after_apply"] == 12 + assert call.kwargs["max_updates_in_single_cmd"] == 64 + assert call.kwargs["set_db"] is True + + +def test_apply_default_apply_fn_lazy_import(dash_utils, monkeypatch): + """When ``apply_fn`` is not provided, the helper should lazy-import + ``gnmi_utils.apply_messages``. We stub that module to confirm it gets + called with the expected positional + keyword arguments.""" + recorded = {} + + def fake_apply_messages(localhost, duthost, ptfhost, messages, dpu_index, + set_db=True, wait_after_apply=5, + max_updates_in_single_cmd=1024): + recorded["args"] = (localhost, duthost, ptfhost, messages, dpu_index) + recorded["kwargs"] = { + "set_db": set_db, + "wait_after_apply": wait_after_apply, + "max_updates_in_single_cmd": max_updates_in_single_cmd, + } + + gnmi_stub = types.ModuleType("gnmi_utils") + gnmi_stub.apply_messages = fake_apply_messages + monkeypatch.setitem(sys.modules, "gnmi_utils", gnmi_stub) + + appliance = {"DASH_APPLIANCE_TABLE:100": {"sip": "10.1.0.5"}} + dash_utils.apply_dash_configs("lh", "dh", "ph", 0, appliance) + + assert recorded["args"][:3] == ("lh", "dh", "ph") + assert recorded["args"][4] == 0 + assert recorded["kwargs"]["set_db"] is True + + +# --------------------------------------------------------------------------- # +# End-to-end check: the canonical phase ordering must split a representative +# fixture's configs into exactly these gNMI batches (using the +# ``test_fnic_basic.py``-style config bundle, which omits METER_RULE): +# 1. GROUP_1 — APPLIANCE +# 2. GROUP_2 — VNET, ROUTING_TYPE, METER_POLICY +# 4. GROUP_4 — TUNNEL, ENI, ROUTE_GROUP, OUTBOUND_PORT_MAP* +# 5. GROUP_5 — ROUTE_RULE, ROUTE, VNET_MAPPING +# 6. GROUP_6 — ENI_ROUTE +# (GROUP_3 / METER_RULE is empty here; fixtures that push METER_RULE +# produce 6 batches.) +# Sentinel configs below share only the DASH table-name part of each +# privatelink_config.py dict; values are placeholders. +# --------------------------------------------------------------------------- # + +_SENTINEL = { + "APPLIANCE_FNIC_CONFIG": {"DASH_APPLIANCE_TABLE:100": {"k": "appl"}}, + "ROUTING_TYPE_PL_CONFIG": {"DASH_ROUTING_TYPE_TABLE:privatelink": {"k": "rtpl"}}, + "ROUTING_TYPE_VNET_CONFIG": {"DASH_ROUTING_TYPE_TABLE:vnet": {"k": "rtvnet"}}, + "VNET_CONFIG": {"DASH_VNET_TABLE:Vnet1": {"k": "vnet"}}, + "ROUTE_GROUP1_CONFIG": {"DASH_ROUTE_GROUP_TABLE:RG1": {"k": "rg"}}, + "METER_POLICY_V4_CONFIG": {"DASH_METER_POLICY_TABLE:MP": {"k": "mp"}}, + "PE_VNET_MAPPING_CONFIG": {"DASH_VNET_MAPPING_TABLE:Vnet1:10.2.0.100": {"k": "pe"}}, + "PE_SUBNET_ROUTE_CONFIG": {"DASH_ROUTE_TABLE:RG1:10.2.0.0/16": {"k": "rpe"}}, + "VM_VNET_MAPPING_CONFIG": {"DASH_VNET_MAPPING_TABLE:Vnet1:10.0.0.11": {"k": "vm"}}, + "VM_SUBNET_ROUTE_CONFIG": {"DASH_ROUTE_TABLE:RG1:10.0.0.0/16": {"k": "rvm"}}, + "VM_VNI_ROUTE_RULE_CONFIG": {"DASH_ROUTE_RULE_TABLE:eni:2001:vm/32": {"k": "rrvm"}}, + "INBOUND_VNI_ROUTE_RULE_CONFIG": {"DASH_ROUTE_RULE_TABLE:eni:100:pe/32": {"k": "rrin"}}, + "TRUSTED_VNI_ROUTE_RULE_CONFIG": {"DASH_ROUTE_RULE_TABLE:eni:800:vm/32": {"k": "rrtr"}}, + "ENI_FNIC_PL_CONFIG": {"DASH_ENI_TABLE:eni-id": {"k": "enipl"}}, + "ENI_FNIC_CONFIG": {"DASH_ENI_TABLE:eni-id": {"k": "enifn"}}, + "ENI_ROUTE_GROUP1_CONFIG": {"DASH_ENI_ROUTE_TABLE:eni-id": {"k": "enirg"}}, +} + + +def _migrated_test_fnic_basic_batches(dash_utils, is_pensando): + """Run the migrated fixture's apply_dash_configs invocation through a + fake apply_fn and return the captured merged messages per call.""" + s = _SENTINEL + route_rule_configs = [] + if not is_pensando: + route_rule_configs = [ + s["VM_VNI_ROUTE_RULE_CONFIG"], + s["INBOUND_VNI_ROUTE_RULE_CONFIG"], + s["TRUSTED_VNI_ROUTE_RULE_CONFIG"], + ] + fake = _fake_apply_fn() + dash_utils.apply_dash_configs( + "lh", "dh", "ph", 0, + s["APPLIANCE_FNIC_CONFIG"], + s["ROUTING_TYPE_PL_CONFIG"], + s["ROUTING_TYPE_VNET_CONFIG"], + s["VNET_CONFIG"], + s["ROUTE_GROUP1_CONFIG"], + s["METER_POLICY_V4_CONFIG"], + s["PE_VNET_MAPPING_CONFIG"], + s["PE_SUBNET_ROUTE_CONFIG"], + s["VM_VNET_MAPPING_CONFIG"], + s["VM_SUBNET_ROUTE_CONFIG"], + *route_rule_configs, + s["ENI_FNIC_PL_CONFIG"], + s["ENI_ROUTE_GROUP1_CONFIG"], + apply_fn=fake, + ) + return [c.args[3] for c in fake.call_args_list] + + +def _expected_test_fnic_basic_batches(is_pensando): + """The canonical phase ordering should yield exactly these batches for + the migrated ``test_fnic_basic.py`` setup.""" + s = _SENTINEL + routes_batch = { + **s["PE_VNET_MAPPING_CONFIG"], + **s["PE_SUBNET_ROUTE_CONFIG"], + **s["VM_VNET_MAPPING_CONFIG"], + **s["VM_SUBNET_ROUTE_CONFIG"], + } + if not is_pensando: + routes_batch.update(s["VM_VNI_ROUTE_RULE_CONFIG"]) + routes_batch.update(s["INBOUND_VNI_ROUTE_RULE_CONFIG"]) + routes_batch.update(s["TRUSTED_VNI_ROUTE_RULE_CONFIG"]) + return [ + # GROUP_1: APPLIANCE + dict(s["APPLIANCE_FNIC_CONFIG"]), + # GROUP_2: VNET + ROUTING_TYPE + METER_POLICY + {**s["ROUTING_TYPE_PL_CONFIG"], **s["ROUTING_TYPE_VNET_CONFIG"], + **s["VNET_CONFIG"], **s["METER_POLICY_V4_CONFIG"]}, + # GROUP_4: ROUTE_GROUP + ENI (GROUP_3 is empty — no METER_RULE here) + {**s["ROUTE_GROUP1_CONFIG"], **s["ENI_FNIC_PL_CONFIG"]}, + # GROUP_5: ROUTES (+ ROUTE_RULE on non-Pensando) + routes_batch, + # GROUP_6: ENI_ROUTE + dict(s["ENI_ROUTE_GROUP1_CONFIG"]), + ] + + +@pytest.mark.parametrize("is_pensando", [False, True], + ids=["non-pensando", "pensando"]) +def test_pilot_migration_matches_expected_phase_batches( + dash_utils, is_pensando): + """The migrated ``test_fnic_basic.py`` setup should produce exactly the + 5 phase-bucketed batches defined above.""" + expected = _expected_test_fnic_basic_batches(is_pensando) + actual = _migrated_test_fnic_basic_batches(dash_utils, is_pensando) + + assert len(actual) == len(expected), ( + "batch count differs: actual={}, expected={}".format( + len(actual), len(expected))) + for i, (act, exp) in enumerate(zip(actual, expected)): + assert set(act) == set(exp), ( + "batch {} key sets differ:\n actual only: {}\n expected only: {}" + .format(i, set(act) - set(exp), set(exp) - set(act))) diff --git a/tests/common/utilities.py b/tests/common/utilities.py index aa0d06ba5a5..de1cf63eaec 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -50,6 +50,8 @@ # Wait 300 seconds because sometime 'interfaces-config' service take 45 seconds to response # interfaces-config service issue track by: https://github.com/sonic-net/sonic-buildimage/issues/19045 FILE_CHANGE_TIMEOUT = 300 +DEFAULT_VRF_NAME = "default" +MGMT_VRF_NAME = "mgmt" NON_USER_CONFIG_TABLES = ["FLEX_COUNTER_TABLE", "ASIC_SENSORS", "LOGGER"] @@ -148,7 +150,7 @@ def wait_until(timeout, interval, delay, condition, *args, **kwargs): try: check_result = condition(*args, **kwargs) - except Exception as e: + except (Exception, pytest.fail.Exception) as e: exc_info = sys.exc_info() details = traceback.format_exception(*exc_info) logger.error( @@ -1738,11 +1740,11 @@ def get_day_of_week_distributed_ports_from_buckets(ports: list, num_buckets: int # Get DoW day_of_week = datetime.now().weekday() logger.info("Day of Week: {} (0=Mon, 6=Sun) - used for port selection".format(day_of_week)) + day_index = datetime.now().toordinal() + rng = random.Random(day_index) # Local RNG: reproducible per day, no global side effects bucket_size = len(ports) // num_buckets remainder = len(ports) % num_buckets - shuffled_ports = list(ports) - random.shuffle(shuffled_ports) selected_ports = [] start_idx = 0 @@ -1752,7 +1754,8 @@ def get_day_of_week_distributed_ports_from_buckets(ports: list, num_buckets: int if current_bucket_size == 0: break end_idx = start_idx + current_bucket_size - bucket_ports = shuffled_ports[start_idx:end_idx] + bucket_ports = ports[start_idx:end_idx] + rng.shuffle(bucket_ports) # Select port based on DoW index (wrapping if bucket is smaller than 7) port_index = day_of_week % len(bucket_ports) selected_ports.append(bucket_ports[port_index]) @@ -1787,3 +1790,25 @@ def testbed_is_multi_vrf(tbinfo): if val: return str(val).lower() == 'true' return False + + +def get_neighbor_exabgp_vm_offset(nbrhosts, tbinfo, neighbor_name): + """Return the vm_offset used to derive a neighbor's exabgp API port. + + The per-neighbor exabgp instances are created from the ORIGINAL topology + offsets. On a converged (multi-VRF) topology those original offsets are + preserved per logical neighbor in ``multi_vrf_data['vm_offset_mapping']`` + (sourced from ``convergence_data['vm_offset_mapping']``), while + ``tbinfo['topo']['properties']['topology']['VMs']`` only carries the + collapsed *prime* offsets. Route-injection helpers that compute + ``EXABGP_BASE_PORT + offset`` must therefore use the per-neighbor original + offset, otherwise they post the announce to the wrong exabgp instance (a + different VRF) and the route never reaches the intended neighbor. + + On stock topologies the neighbor is its own VM and this simply returns the + VM's ``vm_offset``, so behavior is unchanged there. + """ + neighbor = nbrhosts.get(neighbor_name, {}) + if neighbor.get('is_multi_vrf_peer', False): + return neighbor['multi_vrf_data']['vm_offset_mapping'] + return tbinfo['topo']['properties']['topology']['VMs'][neighbor_name]['vm_offset'] diff --git a/tests/conftest.py b/tests/conftest.py index f732253ae1b..f72ef891769 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,7 +54,8 @@ from tests.common.helpers.dut_ports import encode_dut_port_name from tests.common.helpers.dut_utils import encode_dut_and_container_name from tests.common.helpers.parallel_utils import ParallelCoordinator, ParallelStatus, ParallelRunContext -from tests.common.helpers.pfcwd_helper import TrafficPorts, select_test_ports, set_pfc_timers +from tests.common.helpers.pfcwd_helper import TrafficPorts, select_test_ports, set_pfc_timers, \ + is_pfcwd_hw_recovery_enabled from tests.common.system_utils import docker from tests.common.testbed import TestbedInfo from tests.common.utilities import get_inventory_files, wait_until @@ -296,6 +297,9 @@ def pytest_addoption(parser): help="Enable macsec on some links of testbed") parser.addoption("--macsec_profile", action="store", default="all", type=str, help="profile name list in macsec/profile.json") + parser.addoption("--per_interface_macsec", action="store_true", default=False, + help="Layer per-interface MACsec profiles (unique CAK/CKN per port) " + "on top of the base profile for testing") ############################ # QoS options # @@ -465,6 +469,22 @@ def converge_topo_if_needed(config): if not use_converged_peers: logger.info(f"use_converged_peers=False for testbed '{tbname}', skipping converge") return + + # The converged (multi-VRF) peer model is implemented for cEOS neighbors + # only (see ansible/testbed-cli.sh::converge_topo_if_needed and the + # cEOS startup-config templates). For SONiC-VS / cisco / csonic + # neighbors there is no converged render path, so reshaping the in-memory + # topology here would make tbinfo disagree with the actually-deployed + # (unconverged) testbed. Gate on neighbor_type so non-cEOS runs keep the + # historical, byte-identical behavior. + neighbor_type = config.getoption("--neighbor_type") + if neighbor_type not in ("eos", "ceos"): + logger.info( + f"use_converged_peers=True for testbed '{tbname}' but neighbor_type=" + f"'{neighbor_type}' is not cEOS; skipping converge (converged peer " + f"model is cEOS-only)") + return + logger.info(f"use_converged_peers=True for testbed '{tbname}', starting converge...") topo_name = tb_config.get('topo', '').strip() @@ -1076,6 +1096,28 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): return devices neighbor_type = request.config.getoption("--neighbor_type") + neighbor_type_overridden = any( + arg == "--neighbor_type" or arg.startswith("--neighbor_type=") + for arg in request.config.invocation_params.args + ) + # Auto-derive the neighbor type from the testbed's ``vm_type`` field only + # when no ``--neighbor_type`` override was provided on the pytest command + # line. This lets non-EOS testbeds (e.g. cSONiC, vsonic) resolve the correct + # neighbor host class (CsonicHost/SonicHost) while preserving explicit CLI + # overrides. + if not neighbor_type_overridden: + tb_vm_type = tbinfo.get("vm_type") + valid_vm_types = ("eos", "sonic", "cisco", "csonic", "vsonic", "ceos") + if tb_vm_type and tb_vm_type in valid_vm_types: + if tb_vm_type != neighbor_type: + logger.info( + "nbrhosts: deriving neighbor_type='%s' from testbed vm_type " + "(--neighbor_type was not provided)", tb_vm_type) + neighbor_type = tb_vm_type + elif tb_vm_type: + logger.warning( + "nbrhosts: testbed vm_type='%s' is not a recognized neighbor type; " + "falling back to neighbor_type='%s'", tb_vm_type, neighbor_type) if 'VMs' not in tbinfo['topo']['properties']['topology']: logger.info("No VMs exist for this topology: {}".format( tbinfo['topo']['properties']['topology'])) @@ -1112,6 +1154,23 @@ def initial_neighbor(neighbor_name, vm_name, multi_vrf_peer=False, multi_vrf_pri 'multi_vrf_data': multi_vrf_data if multi_vrf_peer else None, } ) + if multi_vrf_peer: + device['host'].bgp_vrf = multi_vrf_data['vrf'] + device['host'].bgp_prime_asn = multi_vrf_data['primary_host_asn'] + device['host'].intf_map = multi_vrf_data['orig_intf_map'] + elif neighbor_type == "csonic": + # cSONiC neighbors are docker-sonic-vs containers accessed via + # "docker exec" (CsonicHost), not over SSH. Handle them before the + # generic "sonic in neighbor_type" branch below, which routes the + # SSH-based SONiC family (sonic, vsonic) to SonicHost. + vm_set_name = tbinfo.get('group-name', '') + container_name = "csonic_{}_{}".format(vm_set_name, vm_name) + device = NeighborDevice( + { + 'host': CsonicHost(container_name), + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] + } + ) elif "sonic" in neighbor_type: device = NeighborDevice( { @@ -1136,15 +1195,6 @@ def initial_neighbor(neighbor_name, vm_name, multi_vrf_peer=False, multi_vrf_pri 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] } ) - elif neighbor_type == "csonic": - vm_set_name = tbinfo.get('group-name', '') - container_name = "csonic_{}_{}".format(vm_set_name, vm_name) - device = NeighborDevice( - { - 'host': CsonicHost(container_name), - 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] - } - ) else: raise ValueError("Unknown neighbor type %s" % (neighbor_type,)) devices[neighbor_name] = device @@ -3750,8 +3800,13 @@ def setup_pfc_test( logger.info("--- Stopping Pfcwd ---") duthost.command("pfcwd stop") - # set poll interval - duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time'])) + # set poll interval (only for software recovery mechanism; HW PFCwd doesn't + # support 'pfcwd interval' since the poll runs in silicon) + if is_pfcwd_hw_recovery_enabled(duthost): + logger.info("--- Hardware recovery mechanism detected - poll interval not supported ---") + else: + logger.info("--- Setting poll interval for software recovery mechanism ---") + duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time'])) # set bulk counter chunk size logger.info("--- Setting bulk counter polling chunk size ---") @@ -3867,6 +3922,34 @@ def setup_connection(request, setup_gnmi_server): channel.close() +def backup_golden_config(duthost, backup_path="/tmp/golden_config_db_backup.json"): + duthost.shell("cp {} {}".format(GOLDEN_CONFIG_DB_PATH, backup_path)) + + +def restore_golden_config(duthost, backup_path="/tmp/golden_config_db_backup.json"): + duthost.shell("cp {} {}".format(backup_path, GOLDEN_CONFIG_DB_PATH)) + + +def update_golden_config_tsa_enabled(duthost, tsa_enabled=True): + """ + @summary: Update golden_config_db.json on the DUT to set tsa_enabled in BGP_DEVICE_GLOBAL. + Handles both multi-asic and single-asic cases. + """ + golden_config_db = json.loads(duthost.shell("cat {}".format(GOLDEN_CONFIG_DB_PATH))['stdout']) + tsa_enabled_str = "true" if tsa_enabled else "false" + + if duthost.sonichost.is_multi_asic: + for asic in duthost.asics: + golden_config_db.setdefault(asic.namespace, {}) \ + .setdefault("BGP_DEVICE_GLOBAL", {}) \ + .setdefault("STATE", {})["tsa_enabled"] = tsa_enabled_str + else: + golden_config_db.setdefault("BGP_DEVICE_GLOBAL", {}) \ + .setdefault("STATE", {})["tsa_enabled"] = tsa_enabled_str + + duthost.copy(content=json.dumps(golden_config_db, indent=4), dest=GOLDEN_CONFIG_DB_PATH) + + @pytest.fixture(scope="module", autouse=True) def restore_golden_config_db(duthost): if file_exists_on_dut(duthost, GOLDEN_CONFIG_DB_PATH_ORI): @@ -4128,6 +4211,14 @@ def yang_validation_check(request, duthosts): logger.info("Skipping YANG validation post-check due to --skip_yang flag") return + for m in request.node.iter_markers(): + if m.name == "skip_check_dut_health": + logger.info( + "Skipping YANG validation: module marked skip_check_dut_health" + ) + yield + return + def run_yang_validation_all(stage): """Run YANG validation on all DUTs and return results""" validation_results = {} diff --git a/tests/copp/test_copp.py b/tests/copp/test_copp.py index b0c3400b04c..e4d9f4930ba 100644 --- a/tests/copp/test_copp.py +++ b/tests/copp/test_copp.py @@ -112,8 +112,9 @@ def test_policer(self, protocol, duthosts, enum_rand_one_per_hwsku_frontend_host # UDLD packet will not be forwarded to DUT if 'UDLD' == protocol: for fanouthost in list(fanouthosts.values()): - if fanouthost.get_fanout_os() == 'sonic' and "arista_7060x6_64pe" in fanouthost.facts["platform"]: - pytest.skip("Skip UDLD test for Arista-7060x6 fanout without UDLD forward support") + if (fanouthost.get_fanout_os() == 'sonic' and fanouthost.facts["platform"] + in ['arista_7060x6_64pe', 'x86_64-nokia_ixr7220_h5_64o-r0', 'x86_64-nokia_ixr7220_h6_64-r0']): + pytest.skip("Skip UDLD test for Arista-7060x6 and Nokia-H5/H6 fanout without UDLD forward support") duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] namespace = DEFAULT_NAMESPACE diff --git a/tests/crm/test_crm.py b/tests/crm/test_crm.py index 94fcb30c918..b469526ef54 100755 --- a/tests/crm/test_crm.py +++ b/tests/crm/test_crm.py @@ -1003,7 +1003,11 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, crm_stats_neighbor_used, crm_stats_neighbor_available = get_crm_stats(get_neighbor_stats, duthost) # Add reachability to the neighbor - if is_cisco_device(duthost): + # Cisco and broadcom-dnx VOQ platforms need the host IP on the interface + # so the neighbor address is in-subnet and gets programmed by orchagent. + needs_host_ip = is_cisco_device(duthost) or \ + duthost.facts.get("platform_asic") == "broadcom-dnx" + if needs_host_ip: asichost.config_ip_intf(crm_interface[0], host, "add") # Add neighbor asichost.shell(neighbor_add_cmd) @@ -1018,7 +1022,7 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, "\"crm_stats_ipv4_neighbor_available\" counter was not decremented") # Remove reachability to the neighbor - if is_cisco_device(duthost): + if needs_host_ip: asichost.config_ip_intf(crm_interface[0], host, "remove") # Remove neighbor asichost.shell(neighbor_del_cmd) diff --git a/tests/dash/configs/privatelink_config.py b/tests/dash/configs/privatelink_config.py index 1a4d958afba..49a8c0c23dc 100644 --- a/tests/dash/configs/privatelink_config.py +++ b/tests/dash/configs/privatelink_config.py @@ -20,6 +20,10 @@ PL_ENCODING_MASK = "::ffff:ffff:ffff:0:0" PL_OVERLAY_SIP = "fd41:108:20:abc:abc::0" PL_OVERLAY_SIP_MASK = "ffff:ffff:ffff:ffff:ffff:ffff::" +PL_ENCODING_IP_ALTERNATE = "fd40::d107:64:ff71:0:0" +PL_ENCODING_MASK_ALTERNATE = "fffe:0:0:ffff:ffff:ffff::" +PL_OVERLAY_SIP_ALTERNATE = "1:108:20::" +PL_OVERLAY_SIP_MASK_ALTERNATE = "1:ffff:ffff::" PL_OVERLAY_DIP = "2603:10e1:100:2::3401:203" PL_OVERLAY_DIP_MASK = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" PL_REDIRECT_OVERLAY_DIP = "2603:10e1:100:2::0" @@ -461,3 +465,21 @@ "port_map": PORT_MAP_1, } } + +DEFAULT_PL_SIP = (PL_ENCODING_IP, PL_ENCODING_MASK, PL_OVERLAY_SIP, PL_OVERLAY_SIP_MASK) +PL_SIP_ALTERNATE = ( + PL_ENCODING_IP_ALTERNATE, + PL_ENCODING_MASK_ALTERNATE, + PL_OVERLAY_SIP_ALTERNATE, + PL_OVERLAY_SIP_MASK_ALTERNATE, +) +PL_SIP_CONFIGS = ( + ENI_FNIC_CONFIG, + ENI_FNIC_PL_CONFIG, + ENI_CONFIG, + PE_VNET_MAPPING_CONFIG, + PE_PLNSG_SINGLE_ENDPOINT_VNET_MAPPING_CONFIG, + PE_PLNSG_MULTI_ENDPOINT_VNET_MAPPING_CONFIG, + PL_REDIRECT_PE_VNET_MAPPING_CONFIG, + PL_REDIRECT_PE_PLNSG_SINGLE_ENDPOINT_VNET_MAPPING_CONFIG, +) diff --git a/tests/dash/conftest.py b/tests/dash/conftest.py index 25f439dd066..aee3a0fe741 100644 --- a/tests/dash/conftest.py +++ b/tests/dash/conftest.py @@ -501,6 +501,31 @@ def dpu_index(request): return request.config.getoption("--dpu_index") +def _apply_pl_sip(sip_params): + encoding_ip, encoding_mask, overlay_sip, overlay_sip_mask = sip_params + pl.PL_ENCODING_IP = encoding_ip + pl.PL_ENCODING_MASK = encoding_mask + pl.PL_OVERLAY_SIP = overlay_sip + pl.PL_OVERLAY_SIP_MASK = overlay_sip_mask + pl_sip_encoding = f"{encoding_ip}/{encoding_mask}" + overlay_sip_prefix = f"{overlay_sip}/{overlay_sip_mask}" + for cfg in pl.PL_SIP_CONFIGS: + for entry in cfg.values(): + if "pl_sip_encoding" in entry: + entry["pl_sip_encoding"] = pl_sip_encoding + if "overlay_sip_prefix" in entry: + entry["overlay_sip_prefix"] = overlay_sip_prefix + + +@pytest.fixture(scope="module", autouse=True) +def configure_pl_sip_for_platform(request): + if "dpuhosts" not in request.fixturenames: + return + dpuhost = request.getfixturevalue("dpuhosts")[request.getfixturevalue("dpu_index")] + sip_params = pl.PL_SIP_ALTERNATE if "bluefield" in dpuhost.facts["asic_type"] else pl.DEFAULT_PL_SIP + _apply_pl_sip(sip_params) + + @pytest.fixture(scope="module", params=[True, False], ids=["single-endpoint", "multi-endpoint"]) def single_endpoint(request): return request.param diff --git a/tests/dash/test_config_churn.py b/tests/dash/test_config_churn.py index 440684574e5..5c4c27aedb9 100644 --- a/tests/dash/test_config_churn.py +++ b/tests/dash/test_config_churn.py @@ -1,3 +1,4 @@ +# flake8: noqa: E231 import logging import time @@ -8,6 +9,7 @@ from gnmi_utils import apply_messages from tests.common import config_reload +from tests.common.dash_utils import apply_dash_configs from tests.common.helpers.assertions import pytest_assert from tests.dash.sairedis_utils import (get_sairedis_line_count, parse_sairedis_changes) @@ -33,44 +35,37 @@ def common_setup_teardown( yield return dpuhost = dpuhosts[dpu_index] - logger.info(pl.ROUTING_TYPE_PL_CONFIG) - - base_config_messages = { - **pl.APPLIANCE_FNIC_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - } - logger.info(base_config_messages) - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_VNET_MAPPING_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG, - } - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) # inbound routing not implemented in Pensando SAI yet, so skip route rule programming + route_rule_configs = [] if "pensando" not in dpuhost.facts["asic_type"]: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - **pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_FNIC_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + route_rule_configs = [ + pl.VM_VNI_ROUTE_RULE_CONFIG, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG, + pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, + ] + + # ``apply_dash_configs`` buckets entries by DASH table name and applies + # them in dependency order (see ``DashPhase`` in ``tests/common/dash_utils.py``): + # GROUP_1 (APPLIANCE) -> GROUP_2 (ROUTING_TYPE/METER_POLICY/OUTBOUND_PORT_MAP/VNET) -> + # GROUP_3 (METER_RULE) -> GROUP_4 (TUNNEL/OUTBOUND_PORT_MAP_RANGE/ENI/ROUTE_GROUP) -> + # GROUP_5 (ROUTE_RULE/ROUTE/VNET_MAPPING) -> GROUP_6 (ENI_ROUTE). + apply_dash_configs( + localhost, duthost, ptfhost, dpuhost.dpu_index, + pl.APPLIANCE_FNIC_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.ROUTING_TYPE_VNET_CONFIG, + pl.VNET_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.METER_POLICY_V4_CONFIG, + pl.PE_VNET_MAPPING_CONFIG, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_VNET_MAPPING_CONFIG, + pl.VM_SUBNET_ROUTE_CONFIG, + *route_rule_configs, + pl.ENI_FNIC_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + ) yield @@ -99,7 +94,7 @@ def test_route_bind_churn(localhost, duthost, ptfhost, dpuhosts, dpu_index): pe_subnet_route_group2_config = { f"DASH_ROUTE_TABLE:{pl.ROUTE_GROUP2}:{pl.PE_CA_SUBNET}": { "routing_type": RoutingType.ROUTING_TYPE_VNET, - "vnet": pl.VNET2, + "vnet": pl.VNET1, "metering_class_or": "2048", "metering_class_and": "4095", } @@ -273,15 +268,24 @@ def test_vnet_churn(localhost, duthost, ptfhost, dpuhosts, dpu_index): **pl.VM_SUBNET_ROUTE_CONFIG, } apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index, False) + apply_messages(localhost, duthost, ptfhost, pl.ROUTE_GROUP1_CONFIG, dpuhost.dpu_index, False) + apply_messages(localhost, duthost, ptfhost, pl.METER_POLICY_V4_CONFIG, dpuhost.dpu_index, False) apply_messages(localhost, duthost, ptfhost, pl.VNET_CONFIG, dpuhost.dpu_index, False) time.sleep(2) new_vni = "3001" vnet_config_v2 = {f"DASH_VNET_TABLE:{pl.VNET1}": {"vni": new_vni, "guid": pl.VNET1_GUID}} + eni_fnic_config_v2 = dict(pl.ENI_FNIC_CONFIG) # deep copy to avoid modifying the original + eni_key = list(eni_fnic_config_v2.keys())[0] + # Update pl_sip_encoding to reflect the new VNI of 3001 + eni_fnic_config_v2[eni_key]["pl_sip_encoding"] = f"::b90b:64:ff71:0:0/{pl.PL_ENCODING_MASK}" + apply_messages(localhost, duthost, ptfhost, vnet_config_v2, dpuhost.dpu_index) + apply_messages(localhost, duthost, ptfhost, pl.ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_messages(localhost, duthost, ptfhost, pl.METER_POLICY_V4_CONFIG, dpuhost.dpu_index) apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) + apply_messages(localhost, duthost, ptfhost, eni_fnic_config_v2, dpuhost.dpu_index) apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) time.sleep(2) diff --git a/tests/dash/test_dash_metering.py b/tests/dash/test_dash_metering.py index ace44764454..d7234fd9fd8 100644 --- a/tests/dash/test_dash_metering.py +++ b/tests/dash/test_dash_metering.py @@ -1,3 +1,4 @@ +# flake8: noqa: E231 import logging import time @@ -11,7 +12,7 @@ from tests.common.helpers.assertions import pytest_assert from configs.privatelink_config import TUNNEL1_ENDPOINT_IPS, TUNNEL2_ENDPOINT_IPS from tests.common import config_reload -from tests.common.dash_utils import verify_tunnel_packets +from tests.common.dash_utils import apply_dash_configs, verify_tunnel_packets from dash_eni_counter_utils import get_eni_counter_oid, get_eni_meter_counters logger = logging.getLogger(__name__) @@ -51,93 +52,60 @@ def common_setup_teardown( yield return dpuhost = dpuhosts[dpu_index] - logger.info(pl.ROUTING_TYPE_PL_CONFIG) if single_endpoint: tunnel_config = pl.TUNNEL1_CONFIG - else: - tunnel_config = pl.TUNNEL2_CONFIG - - base_config_messages = { - **pl.APPLIANCE_FNIC_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.ROUTE_GROUP2_CONFIG, - **pl.ROUTE_GROUP3_CONFIG, - **tunnel_config, - } - logger.info(base_config_messages) - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - if single_endpoint: rg1_vm_subnet_route_config = pl.VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT rg2_vm_subnet_route_config = pl.RG2_VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT rg3_vm_subnet_route_config = pl.RG3_VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT else: + tunnel_config = pl.TUNNEL2_CONFIG rg1_vm_subnet_route_config = pl.VM_SUBNET_ROUTE_WITH_TUNNEL_MULTI_ENDPOINT rg2_vm_subnet_route_config = pl.RG2_VM_SUBNET_ROUTE_WITH_TUNNEL_MULTI_ENDPOINT rg3_vm_subnet_route_config = pl.RG3_VM_SUBNET_ROUTE_WITH_TUNNEL_MULTI_ENDPOINT - # Route-Group1 rule creation - route_messages = { - **pl.PE_SUBNET_ROUTE_CONFIG, - **rg1_vm_subnet_route_config - } - logger.info(route_messages) - apply_messages(localhost, duthost, ptfhost, route_messages, dpuhost.dpu_index) - - # Route-Group2 rule creation - route_messages = { - **pl.METERCLASSOR_PE_SUBNET_ROUTE_CONFIG, - **rg2_vm_subnet_route_config, - } - logger.info(route_messages) - apply_messages(localhost, duthost, ptfhost, route_messages, dpuhost.dpu_index) - - # Route-Group3 rule creation - route_messages = { - **pl.METERCLASSAND_PE_SUBNET_ROUTE_CONFIG, - **rg3_vm_subnet_route_config, - } - logger.info(route_messages) - apply_messages(localhost, duthost, ptfhost, route_messages, dpuhost.dpu_index) - # inbound routing not implemented in Pensando SAI yet, so skip route rule programming + route_rule_configs = [] if 'pensando' not in dpuhost.facts['asic_type']: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - **pl.TRUSTED_VNI_ROUTE_RULE_CONFIG - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_FNIC_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + route_rule_configs = [ + pl.VM_VNI_ROUTE_RULE_CONFIG, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG, + pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, + ] + + # ``apply_dash_configs`` buckets entries by DASH table name and applies + # them in dependency order (see ``DashPhase`` in ``tests/common/dash_utils.py``): + # GROUP_1 (APPLIANCE) -> GROUP_2 (ROUTING_TYPE/METER_POLICY/OUTBOUND_PORT_MAP/VNET) -> + # GROUP_3 (METER_RULE) -> GROUP_4 (TUNNEL/OUTBOUND_PORT_MAP_RANGE/ENI/ROUTE_GROUP) -> + # GROUP_5 (ROUTE_RULE/ROUTE/VNET_MAPPING) -> GROUP_6 (ENI_ROUTE). + apply_dash_configs( + localhost, duthost, ptfhost, dpuhost.dpu_index, + pl.APPLIANCE_FNIC_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.ROUTING_TYPE_VNET_CONFIG, + pl.VNET_CONFIG, + pl.METER_POLICY_V4_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.ROUTE_GROUP2_CONFIG, + pl.ROUTE_GROUP3_CONFIG, + tunnel_config, + pl.PE_SUBNET_ROUTE_CONFIG, + rg1_vm_subnet_route_config, + pl.METERCLASSOR_PE_SUBNET_ROUTE_CONFIG, + rg2_vm_subnet_route_config, + pl.METERCLASSAND_PE_SUBNET_ROUTE_CONFIG, + rg3_vm_subnet_route_config, + *route_rule_configs, + pl.METER_RULE2_V4_CONFIG, + pl.ENI_FNIC_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + ) yield # Route rule removal is broken so config reload to cleanup for now # https://github.com/sonic-net/sonic-buildimage/issues/23590 config_reload(dpuhost, safe_reload=True, yang_validate=False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_TRUSTED_VNI_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index, False) @pytest.mark.parametrize("metering_tc", ['ENI_METERPOLICY_HIT', 'MAPPING_METERCLASS_HIT', diff --git a/tests/dash/test_dash_privatelink.py b/tests/dash/test_dash_privatelink.py index 6a9c6d444f6..ab4acce2326 100644 --- a/tests/dash/test_dash_privatelink.py +++ b/tests/dash/test_dash_privatelink.py @@ -7,9 +7,9 @@ import ptf.packet as scapy from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF, REMOTE_PTF_SEND_INTF from constants import VXLAN_UDP_BASE_SRC_PORT, VXLAN_UDP_SRC_PORT_MASK -from gnmi_utils import apply_messages from packets import outbound_pl_packets, inbound_pl_packets from tests.common.config_reload import config_reload +from tests.common.dash_utils import apply_dash_configs logger = logging.getLogger(__name__) @@ -39,53 +39,38 @@ def common_setup_teardown( if skip_config: return dpuhost = dpuhosts[dpu_index] - logger.info(pl.ROUTING_TYPE_PL_CONFIG) - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(base_config_messages) - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } + # ``INBOUND_VNI_ROUTE_RULE_CONFIG`` is only programmed on Bluefield DPUs; + # on other platforms ROUTE_RULE entries are skipped at the source. + bluefield_route_rule_configs = [] if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + bluefield_route_rule_configs = [pl.INBOUND_VNI_ROUTE_RULE_CONFIG] + + # ``apply_dash_configs`` buckets entries by DASH table name and applies + # them in dependency order (see ``DashPhase`` in ``tests/common/dash_utils.py``): + # GROUP_1 (APPLIANCE) -> GROUP_2 (ROUTING_TYPE/METER_POLICY/OUTBOUND_PORT_MAP/VNET) -> + # GROUP_3 (METER_RULE) -> GROUP_4 (TUNNEL/OUTBOUND_PORT_MAP_RANGE/ENI/ROUTE_GROUP) -> + # GROUP_5 (ROUTE_RULE/ROUTE/VNET_MAPPING) -> GROUP_6 (ENI_ROUTE). + apply_dash_configs( + localhost, duthost, ptfhost, dpuhost.dpu_index, + pl.APPLIANCE_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.VNET_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.METER_POLICY_V4_CONFIG, + pl.PE_VNET_MAPPING_CONFIG, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_SUBNET_ROUTE_CONFIG, + *bluefield_route_rule_configs, + pl.METER_RULE1_V4_CONFIG, + pl.METER_RULE2_V4_CONFIG, + pl.ENI_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + ) yield config_reload(dpuhost, safe_reload=True, yang_validate=False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index, False) @pytest.mark.parametrize("encap_proto", ["vxlan", "gre"]) diff --git a/tests/dash/test_fnic.py b/tests/dash/test_fnic.py index fc39ca88358..ef70b47e024 100644 --- a/tests/dash/test_fnic.py +++ b/tests/dash/test_fnic.py @@ -1,16 +1,14 @@ -import logging - import configs.privatelink_config as pl import ptf.testutils as testutils import ptf.packet as scapy import pytest +import logging from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF, REMOTE_PTF_SEND_INTF -from gnmi_utils import apply_messages from packets import rand_udp_port_packets from tests.common.helpers.assertions import pytest_assert from configs.privatelink_config import TUNNEL1_ENDPOINT_IPS, TUNNEL2_ENDPOINT_IPS from tests.common import config_reload -from tests.common.dash_utils import verify_tunnel_packets +from tests.common.dash_utils import apply_dash_configs, verify_tunnel_packets logger = logging.getLogger(__name__) @@ -49,72 +47,53 @@ def common_setup_teardown( yield return dpuhost = dpuhosts[dpu_index] - logger.info(pl.ROUTING_TYPE_PL_CONFIG) if single_endpoint: tunnel_config = pl.TUNNEL1_CONFIG - else: - tunnel_config = pl.TUNNEL2_CONFIG - - base_config_messages = { - **pl.APPLIANCE_FNIC_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - **tunnel_config, - } - logger.info(base_config_messages) - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - if single_endpoint: vm_subnet_route_config = pl.VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT else: + tunnel_config = pl.TUNNEL2_CONFIG vm_subnet_route_config = pl.VM_SUBNET_ROUTE_WITH_TUNNEL_MULTI_ENDPOINT - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_VNET_MAPPING_CONFIG, - **vm_subnet_route_config - } - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) # inbound routing not implemented in Pensando SAI yet, so skip route rule programming + route_rule_configs = [] if 'pensando' not in dpuhost.facts['asic_type']: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - **pl.TRUSTED_VNI_ROUTE_RULE_CONFIG - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_FNIC_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + route_rule_configs = [ + pl.VM_VNI_ROUTE_RULE_CONFIG, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG, + pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, + ] + + # ``apply_dash_configs`` buckets entries by DASH table name and applies + # them in dependency order (see ``DashPhase`` in ``tests/common/dash_utils.py``): + # GROUP_1 (APPLIANCE) -> GROUP_2 (ROUTING_TYPE/METER_POLICY/OUTBOUND_PORT_MAP/VNET) -> + # GROUP_3 (METER_RULE) -> GROUP_4 (TUNNEL/OUTBOUND_PORT_MAP_RANGE/ENI/ROUTE_GROUP) -> + # GROUP_5 (ROUTE_RULE/ROUTE/VNET_MAPPING) -> GROUP_6 (ENI_ROUTE). + apply_dash_configs( + localhost, duthost, ptfhost, dpuhost.dpu_index, + pl.APPLIANCE_FNIC_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.ROUTING_TYPE_VNET_CONFIG, + pl.VNET_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.METER_POLICY_V4_CONFIG, + tunnel_config, + pl.PE_VNET_MAPPING_CONFIG, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_VNET_MAPPING_CONFIG, + vm_subnet_route_config, + *route_rule_configs, + pl.METER_RULE1_V4_CONFIG, + pl.METER_RULE2_V4_CONFIG, + pl.ENI_FNIC_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + ) yield # Route rule removal is broken so config reload to cleanup for now # https://github.com/sonic-net/sonic-buildimage/issues/23590 config_reload(dpuhost, safe_reload=True, yang_validate=False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, pl.ENI_TRUSTED_VNI_CONFIG, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index, False) - # apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index, False) @pytest.mark.parametrize("encap_proto", ["vxlan", "gre"]) diff --git a/tests/dash/test_plnsg.py b/tests/dash/test_plnsg.py index 88fac107a88..3368aa6cd26 100644 --- a/tests/dash/test_plnsg.py +++ b/tests/dash/test_plnsg.py @@ -10,9 +10,8 @@ VXLAN_UDP_BASE_SRC_PORT, VXLAN_UDP_SRC_PORT_MASK, ) -from gnmi_utils import apply_messages from packets import inbound_pl_packets, plnsg_packets -from tests.common.dash_utils import verify_tunnel_packets +from tests.common.dash_utils import apply_dash_configs, verify_tunnel_packets from tests.common.helpers.assertions import pytest_assert as pt_assert import ptf.packet as scapy @@ -42,59 +41,44 @@ def config_setup_teardown( yield return dpuhost = dpuhosts[dpu_index] - logger.info(pl.ROUTING_TYPE_PL_CONFIG) if single_endpoint: tunnel_config = pl.TUNNEL3_CONFIG - else: - tunnel_config = pl.TUNNEL4_CONFIG - - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.VNET2_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - **tunnel_config, - } - logger.info(base_config_messages) - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - if single_endpoint: vnet_mapping_config = pl.PE_PLNSG_SINGLE_ENDPOINT_VNET_MAPPING_CONFIG else: + tunnel_config = pl.TUNNEL4_CONFIG vnet_mapping_config = pl.PE_PLNSG_MULTI_ENDPOINT_VNET_MAPPING_CONFIG - route_and_mapping_messages = { - **vnet_mapping_config, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG, - } - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) + route_rule_configs = [] if 'pensando' not in dpuhost.facts['asic_type']: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + route_rule_configs = [ + pl.VM_VNI_ROUTE_RULE_CONFIG, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG, + ] + + # ``apply_dash_configs`` buckets entries by DASH table name and applies + # them in dependency order (see ``DashPhase`` in ``tests/common/dash_utils.py``): + # GROUP_1 (APPLIANCE) -> GROUP_2 (ROUTING_TYPE/METER_POLICY/OUTBOUND_PORT_MAP/VNET) -> + # GROUP_3 (METER_RULE) -> GROUP_4 (TUNNEL/OUTBOUND_PORT_MAP_RANGE/ENI/ROUTE_GROUP) -> + # GROUP_5 (ROUTE_RULE/ROUTE/VNET_MAPPING) -> GROUP_6 (ENI_ROUTE). + apply_dash_configs( + localhost, duthost, ptfhost, dpuhost.dpu_index, + pl.APPLIANCE_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.ROUTING_TYPE_VNET_CONFIG, + pl.VNET_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.METER_POLICY_V4_CONFIG, + tunnel_config, + vnet_mapping_config, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_SUBNET_ROUTE_CONFIG, + *route_rule_configs, + pl.METER_RULE1_V4_CONFIG, + pl.METER_RULE2_V4_CONFIG, + pl.ENI_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + ) yield diff --git a/tests/database/test_state_db_flush.py b/tests/database/test_state_db_flush.py new file mode 100644 index 00000000000..a5731c0efd0 --- /dev/null +++ b/tests/database/test_state_db_flush.py @@ -0,0 +1,131 @@ +""" +Test STATE_DB table cleanup during swss cold restart. + +Verifies that tables owned by orchagent in STATE_DB are properly cleaned up +when swss is restarted (non-warm-boot). This ensures stale state from a +previous orchagent run does not persist across service restarts. +""" +import logging + +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.common.platform.processes_utils import wait_critical_processes + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('t0', 't1', 't2', 'lt2', 'ft2'), + pytest.mark.disable_loganalyzer +] + +# All STATE_DB tables that swss.sh cleans up on cold restart. +# Sentinel keys use a fake sub-key that orchagent will never recreate, +# so their absence after restart proves the cleanup ran. +STATE_DB_CLEANUP_TABLES = [ + "PORT_TABLE", + "MGMT_PORT_TABLE", + "VLAN_TABLE", + "VLAN_MEMBER_TABLE", + "LAG_TABLE", + "LAG_MEMBER_TABLE", + "INTERFACE_TABLE", + "MIRROR_SESSION", + "VRF_TABLE", + "FDB_TABLE", + "FG_ROUTE_TABLE", + "BUFFER_POOL", + "BUFFER_PROFILE", + "MUX_CABLE_TABLE", + "ADVERTISE_NETWORK_TABLE", + "VXLAN_TUNNEL_TABLE", + "VNET_ROUTE", + "MACSEC_PORT_TABLE", + "MACSEC_INGRESS_SA_TABLE", + "MACSEC_EGRESS_SA_TABLE", + "MACSEC_INGRESS_SC_TABLE", + "MACSEC_EGRESS_SC_TABLE", + "VRF_OBJECT_TABLE", + "VNET_MONITOR_TABLE", + "BFD_SESSION_TABLE", + "SYSTEM_NEIGH_TABLE", + "FABRIC_PORT_TABLE", + "TUNNEL_DECAP_TABLE", + "TUNNEL_DECAP_TERM_TABLE", + "HIGH_FREQUENCY_TELEMETRY_SESSION_TABLE", + "PROCESS_HEALTH", +] + +SENTINEL_SUBKEY = "__test_sentinel__" + + +def _sentinel_key(table): + """Return a sentinel STATE_DB key for the given table.""" + return "{}|{}".format(table, SENTINEL_SUBKEY) + + +def _is_service_hitting_start_limit(duthost, container_name): + """Check if a service is hitting the systemd start-limit.""" + result = duthost.shell( + "sudo systemctl status {}.service | grep 'Active'".format(container_name), + module_ignore_errors=True + ) + for line in result["stdout_lines"]: + if "start-limit-hit" in line: + return True + return False + + +def test_state_db_cleanup_after_swss_restart(duthosts, enum_rand_one_per_hwsku_frontend_hostname, + enum_frontend_asic_index): + """ + Verify all orchagent-owned STATE_DB tables are cleaned up during swss cold restart. + + Steps: + 1. Populate a sentinel entry in every STATE_DB table that swss.sh cleans. + 2. Confirm all sentinel entries exist. + 3. Cold-restart swss for the given ASIC. + 4. Verify every sentinel entry is removed from STATE_DB. + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + asichost = duthost.asic_instance(enum_frontend_asic_index) + service_name = asichost.get_service_name("swss") + db_cli = "{} STATE_DB".format(asichost.sonic_db_cli) + + # Populate sentinel keys for all tables + for table in STATE_DB_CLEANUP_TABLES: + key = _sentinel_key(table) + logger.info("Setting sentinel STATE_DB entry: {} (asic {})".format(key, enum_frontend_asic_index)) + duthost.shell('{} HSET "{}" "test_field" "test_value"'.format(db_cli, key)) + + # Verify all sentinel keys were created + missing = [] + for table in STATE_DB_CLEANUP_TABLES: + result = duthost.shell('{} EXISTS "{}"'.format(db_cli, _sentinel_key(table))) + if result["stdout"].strip() != "1": + missing.append(table) + pytest_assert(not missing, + "Failed to create sentinel entries for tables: {}".format(missing)) + + # Cold-restart swss for this ASIC + logger.info("Cold-restarting {} on asic {}".format(service_name, enum_frontend_asic_index)) + duthost.shell("sudo systemctl reset-failed {}".format(service_name), module_ignore_errors=True) + duthost.shell("sudo systemctl restart {}".format(service_name)) + + for container in duthost.get_default_critical_services_list(): + if _is_service_hitting_start_limit(duthost, container): + logger.info("{} hit start limit, resetting".format(container)) + duthost.shell("sudo systemctl reset-failed {}.service".format(container)) + duthost.shell("sudo systemctl start {}.service".format(container)) + + wait_critical_processes(duthost) + + # Verify all sentinel keys are cleaned up + stale = [] + for table in STATE_DB_CLEANUP_TABLES: + result = duthost.shell('{} EXISTS "{}"'.format(db_cli, _sentinel_key(table))) + if result["stdout"].strip() == "1": + stale.append(table) + pytest_assert(not stale, + "STATE_DB entries not cleaned up after swss cold restart (asic {}): {}".format( + enum_frontend_asic_index, stale)) diff --git a/tests/dhcp_relay/test_dhcp_relay.py b/tests/dhcp_relay/test_dhcp_relay.py index 398f9d505c7..c900e936ce4 100644 --- a/tests/dhcp_relay/test_dhcp_relay.py +++ b/tests/dhcp_relay/test_dhcp_relay.py @@ -715,3 +715,49 @@ def test_dhcp_relay_monitor_checksum_validation(ptfhost, dut_dhcp_relay_data, va except LogAnalyzerError as err: logger.error("Unable to find expected log in syslog") raise err + + +def test_dhcp_broadcast_not_flooded(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, + testing_config, relay_agent): + """Verify DHCP broadcast packets are trapped to CPU and not L2-flooded to other VLAN member ports. + + The COPP trap_action for DHCP is 'trap' (SAI_PACKET_ACTION_TRAP), meaning packets + are sent exclusively to CPU and removed from the forwarding pipeline. This test + sends DHCP Discover and Request broadcasts from a client port and asserts that + no copy of the original broadcast appears on any other VLAN member port. + """ + testing_mode, duthost = testing_config + + if duthost.facts.get("asic_type") == "vs": + pytest.skip("VS/KVM dataplane does not enforce SAI_PACKET_ACTION_TRAP removal semantics; " + "broadcasts are L2-flooded even when COPP traps them to CPU") + + for dhcp_relay in dut_dhcp_relay_data: + if not dhcp_relay['other_client_ports']: + pytest.skip("Need at least two VLAN member ports to verify non-flooding behavior") + + ptf_runner(ptfhost, + "ptftests", + "dhcp_relay_test.DHCPBroadcastNotFloodedTest", + platform_dir="ptftests", + params={"hostname": duthost.hostname, + "client_port_index": dhcp_relay['client_iface']['port_idx'], + "other_client_port": repr(dhcp_relay['other_client_ports']), + "client_iface_alias": str(dhcp_relay['client_iface']['alias']), + "leaf_port_indices": repr(dhcp_relay['uplink_port_indices']), + "num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']), + "server_ip": dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'], + "relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']), + "relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']), + "relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']), + "dest_mac_address": BROADCAST_MAC, + "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, + "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], + "uplink_mac": str(dhcp_relay['uplink_mac']), + "testing_mode": testing_mode, + "kvm_support": True, + "relay_agent": relay_agent, + "downlink_vlan_iface_name": str(dhcp_relay['downlink_vlan_iface']['name'])}, + log_file=("/tmp/dhcp_relay_test.DHCPBroadcastNotFloodedTest.{}.log" + .format(dhcp_relay["downlink_vlan_iface"]["name"])), + is_python3=True) diff --git a/tests/fdb/test_fdb_flush.py b/tests/fdb/test_fdb_flush.py index 4815dd77a81..591d24a55a7 100644 --- a/tests/fdb/test_fdb_flush.py +++ b/tests/fdb/test_fdb_flush.py @@ -264,7 +264,9 @@ def checkDutCorefiles(self, duthost, is_before_test): self.curr_exist_cores = existing_core_dumps def create_fdb_oper_files(self, duthost): - mac_addresses = ["00-11-22-33-55-66", "00-11-22-33-55-67", "00-11-22-33-55-68"] + # Keep static MACs on 00:11:22:33:44:XX to avoid collision with PTF dynamic MACs (00:11:22:33:55:XX). + # Use 44 in the 5th octet; dynamic FDB MACs use 55 and can collide on larger testbeds. + mac_addresses = ["00-11-22-33-44-66", "00-11-22-33-44-67", "00-11-22-33-44-68"] vlan_id = 1000 fdb_static_set = [] fdb_static_del = [] diff --git a/tests/fib/test_fib.py b/tests/fib/test_fib.py index 5fce063f537..35aace4389b 100644 --- a/tests/fib/test_fib.py +++ b/tests/fib/test_fib.py @@ -145,7 +145,7 @@ def map_ptf_ports_to_dut_port(ptf_ports, all_dut_port_indices): return ethernet_ports_with_asic -def filter_ports(all_port_indices, tbinfo): +def filter_ports(all_port_indices, tbinfo, is_chassis): """ Filter PTF ports that need to be skipped while picking up src_port for ptf traffic test. @@ -161,7 +161,7 @@ def filter_ports(all_port_indices, tbinfo): # Note: this filteration is useful for multilinecard DUTs to make sure incoming traffic is # landing on a different linecard; NA for pizza boxes - if tbinfo['topo']['type'] != 't2' or 't2_single_node' in tbinfo['topo']['name']: + if tbinfo['topo']['type'] != 't2' or not is_chassis: return [] # Collect all port indices (keys) from all_port_indices @@ -171,7 +171,7 @@ def filter_ports(all_port_indices, tbinfo): def get_port_and_portchannel_members(port_name, all_port_indices, duts_minigraph_facts, - upstream_lc, tbinfo): + upstream_lc, tbinfo, is_chassis): """ Get PTF port indices for a port and all its port channel members (if applicable). @@ -186,7 +186,7 @@ def get_port_and_portchannel_members(port_name, all_port_indices, duts_minigraph """ # for T2(except UT2) topologies, no need to append, as we are already filtering out the whole upstream lc ports - if tbinfo['topo']['type'] == 't2' and 't2_single_node' not in tbinfo['topo']['name']: + if (tbinfo['topo']['type'] == 't2') and is_chassis: return [] # Search all ASICs for port channel information @@ -868,7 +868,8 @@ def test_ecmp_group_member_flap( all_port_indices = get_all_ptf_port_indices_from_mg_facts(duts_minigraph_facts[upstream_lc]) nh_dut_ports = map_ptf_ports_to_dut_port(nh_ptf_ports, all_port_indices) - filtered_ports = filter_ports(all_port_indices, tbinfo) + is_chassis = duthosts[0].get_facts().get("modular_chassis") + filtered_ports = filter_ports(all_port_indices, tbinfo, is_chassis) logging.info("nh_dut_ports: {}".format(nh_dut_ports)) logging.info("filtered_ports: {}".format(filtered_ports)) @@ -924,7 +925,7 @@ def test_ecmp_group_member_flap( # Get all PTF ports for the port and its port channel members (if applicable) ptf_ports_to_filter = get_port_and_portchannel_members( - nh_dut_ports[port_index_to_shut][1], all_port_indices, duts_minigraph_facts, upstream_lc, tbinfo) + nh_dut_ports[port_index_to_shut][1], all_port_indices, duts_minigraph_facts, upstream_lc, tbinfo, is_chassis) # Add them to filtered_ports filtered_ports.extend(ptf_ports_to_filter) diff --git a/tests/generic_config_updater/test_lo_interface.py b/tests/generic_config_updater/test_lo_interface.py index a13331ba030..25524118e6b 100644 --- a/tests/generic_config_updater/test_lo_interface.py +++ b/tests/generic_config_updater/test_lo_interface.py @@ -385,6 +385,7 @@ def test_lo_interface_tc1_suite(duthosts, rand_one_dut_front_end_hostname, cfg_f lo_interface_tc1_remove(duthost, lo_intf) +@pytest.mark.enable_monit_refresh def test_lo_interface_tc2_vrf_change(duthosts, rand_one_dut_front_end_hostname, lo_intf): """ Replace lo interface vrf diff --git a/tests/generic_config_updater/test_monitor_config.py b/tests/generic_config_updater/test_monitor_config.py index 5e5d34200c0..7c8ed25157b 100644 --- a/tests/generic_config_updater/test_monitor_config.py +++ b/tests/generic_config_updater/test_monitor_config.py @@ -10,6 +10,7 @@ pytestmark = [ pytest.mark.topology('any'), + pytest.mark.enable_monit_refresh, ] logger = logging.getLogger(__name__) @@ -29,6 +30,8 @@ def is_policer_supported(duthost): platform = duthost.facts.get('platform', '') if platform.startswith("x86_64-arista_7060x6"): return False + if platform.startswith("x86_64-nokia_ixr7220"): + return False return True diff --git a/tests/generic_config_updater/test_vlan_interface.py b/tests/generic_config_updater/test_vlan_interface.py index a4bdc0f588a..9c5a5b94849 100644 --- a/tests/generic_config_updater/test_vlan_interface.py +++ b/tests/generic_config_updater/test_vlan_interface.py @@ -413,6 +413,7 @@ def vlan_interface_tc1_remove(duthost, vlan_info): delete_tmpfile(duthost, tmpfile) +@pytest.mark.enable_monit_refresh def test_vlan_interface_tc1_suite(rand_selected_dut, vlan_info, loganalyzer, tbinfo, duthost): if loganalyzer: if tbinfo["topo"]["name"] == "m0-2vlan": diff --git a/tests/gnmi/conftest.py b/tests/gnmi/conftest.py index f6649a4ff0e..ac99b06445c 100755 --- a/tests/gnmi/conftest.py +++ b/tests/gnmi/conftest.py @@ -3,7 +3,8 @@ from tests.common.helpers.assertions import pytest_require as pyrequire from tests.common.helpers.dut_utils import check_container_state -from tests.gnmi.helper import gnmi_container, apply_cert_config, recover_cert_config +from tests.common.helpers.gnmi_utils import gnmi_container +from tests.gnmi.helper import apply_cert_config, recover_cert_config from tests.gnmi.helper import GNMI_SERVER_START_WAIT_TIME, check_ntp_sync_status from tests.common.gu_utils import create_checkpoint, rollback from tests.common.helpers.gnmi_utils import create_revoked_cert_and_crl, create_gnmi_certs, \ @@ -15,6 +16,24 @@ logger = logging.getLogger(__name__) SETUP_ENV_CP = "test_setup_checkpoint" +VRF_SCENARIOS = [ + {"name": "default_1", "vrf": None, "description": "Default (no VRF)"}, +] + + +@pytest.fixture(scope="module", params=VRF_SCENARIOS, ids=lambda scenario: f"vrf_{scenario['name']}") +def vrf_config(request): + return request.param + + +@pytest.fixture(scope="module", autouse=True) +def setup_vrf_configuration(vrf_config): + """ + This fixture runs before setup_gnmi_server to ensure VRF config is in place. + It gets overridden in tests that parameterize the gNMI server VRF binding. + """ + return vrf_config + @pytest.fixture(scope="module") def setup_gnmi_ntp_client_server(duthosts, rand_one_dut_hostname, ptfhost): @@ -42,7 +61,7 @@ def setup_gnmi_ntp_client_server(duthosts, rand_one_dut_hostname, ptfhost): @pytest.fixture(scope="module") -def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost): +def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost, vrf_config, setup_vrf_configuration): ''' Setup GNMI server with client certificates ''' @@ -53,10 +72,10 @@ def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost): check_container_state(duthost, gnmi_container(duthost), should_be_running=True), "Test was not supported on devices which do not support GNMI!") - create_gnmi_certs(duthost, localhost, ptfhost) + create_gnmi_certs(duthost, localhost, ptfhost, dut_ip=vrf_config.get("dut_ip")) create_checkpoint(duthost, SETUP_ENV_CP) - stopped_programs = apply_cert_config(duthost) + stopped_programs = apply_cert_config(duthost, vrf_config.get("vrf")) yield diff --git a/tests/gnmi/helper.py b/tests/gnmi/helper.py index 5a25ab8ecdb..b996eeba87c 100755 --- a/tests/gnmi/helper.py +++ b/tests/gnmi/helper.py @@ -6,7 +6,6 @@ from tests.common.platform.device_utils import get_dpu_ip, get_dpu_port from tests.common.helpers.gnmi_utils import GNMIEnvironment, add_gnmi_client_common_name, del_gnmi_client_common_name, \ dump_gnmi_log, dump_system_status -from tests.common.helpers.gnmi_utils import gnmi_container # noqa: F401 from tests.common.helpers.ntp_helper import NtpDaemon, get_ntp_daemon_in_use # noqa: F401 from tests.common.helpers.dut_utils import check_container_state @@ -19,7 +18,12 @@ GNMI_SERVER_START_WAIT_TIME = 15 -def apply_cert_config(duthost): +def is_mgmt_vrf_enabled(duthost): + res = duthost.shell('sudo sonic-db-cli CONFIG_DB HGET "MGMT_VRF_CONFIG|vrf_global" "mgmtVrfEnabled"')["stdout"] + return res == "true" + + +def apply_cert_config(duthost, vrf_name=None): env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) # Get subtype cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] @@ -50,6 +54,8 @@ def apply_cert_config(duthost): dut_command += "--enable_crl=true " if subtype == 'SmartSwitch': dut_command += "--zmq_address=tcp://127.0.0.1:8100 " + if vrf_name: + dut_command += "--gnmi_vrf %s " % vrf_name dut_command += "--ca_crt /etc/sonic/telemetry/gnmiCA.pem -gnmi_native_write=true -v=10 >/root/gnmi.log 2>&1 &\"" duthost.shell(dut_command) @@ -193,7 +199,7 @@ def check_system_time_sync(duthost): return False -def gnmi_set(duthost, ptfhost, delete_list, update_list, replace_list, cert=None): +def gnmi_set(duthost, ptfhost, delete_list, update_list, replace_list, cert=None, ip=None): """ Send GNMI set request with GNMI client @@ -207,7 +213,7 @@ def gnmi_set(duthost, ptfhost, delete_list, update_list, replace_list, cert=None Returns: """ env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) - ip = duthost.mgmt_ip + ip = ip or duthost.mgmt_ip port = env.gnmi_port cmd = '/root/env-python3/bin/python /root/gnxi/gnmi_cli_py/py_gnmicli.py ' cmd += '--timeout 30 ' @@ -276,7 +282,7 @@ def gnmi_set(duthost, ptfhost, delete_list, update_list, replace_list, cert=None raise Exception(f"py_gnmicli failed rc={rc}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}") -def gnmi_get(duthost, ptfhost, path_list): +def gnmi_get(duthost, ptfhost, path_list, ip=None): """ Send GNMI get request with GNMI client @@ -289,7 +295,7 @@ def gnmi_get(duthost, ptfhost, path_list): msg_list: list for get result """ env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) - ip = duthost.mgmt_ip + ip = ip or duthost.mgmt_ip port = env.gnmi_port cmd = '/root/env-python3/bin/python /root/gnxi/gnmi_cli_py/py_gnmicli.py ' cmd += '--timeout 30 ' @@ -325,7 +331,7 @@ def gnmi_get(duthost, ptfhost, path_list): # py_gnmicli does not fully support POLLING mode # Use gnmi_cli instead -def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count): +def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count, ip=None, vrf_name=None): """ Send GNMI subscribe request with GNMI client @@ -335,6 +341,9 @@ def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count): path_list: list for get path interval_ms: interval, unit is ms count: update count + ip: server IP to connect to (defaults to duthost.mgmt_ip) + vrf_name: when set, run gnmi_cli on the DUT inside the given VRF + (using `ip vrf exec`) instead of inside the gnmi container. Returns: msg: gnmi client output @@ -343,12 +352,18 @@ def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count): logger.error("path_list is None") return "", "" env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) - dut_facts = duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'] - ip = f"[{duthost.mgmt_ip}]" if dut_facts.get('is_mgmt_ipv6_only', False) else duthost.mgmt_ip + if ip is None: + dut_facts = duthost.dut_basic_facts()['ansible_facts']['dut_basic_facts'] + ip = f"[{duthost.mgmt_ip}]" if dut_facts.get('is_mgmt_ipv6_only', False) else duthost.mgmt_ip port = env.gnmi_port interval = interval_ms / 1000.0 - # Run gnmi_cli in gnmi container as workaround - cmd = "docker exec %s gnmi_cli -client_types=gnmi -a %s:%s " % (env.gnmi_container, ip, port) + # For a non-default VRF the gnmi container does not have `ip vrf exec` + # privileges, so run gnmi_cli on the DUT host in the target VRF instead. + if vrf_name and vrf_name != "default": + cmd = "sudo ip vrf exec %s /tmp/gnmi_cli -client_types=gnmi -a %s:%s " % (vrf_name, ip, port) + else: + # Run gnmi_cli in gnmi container as workaround + cmd = "docker exec %s gnmi_cli -client_types=gnmi -a %s:%s " % (env.gnmi_container, ip, port) cmd += "-client_crt /etc/sonic/telemetry/gnmiclient.crt " cmd += "-client_key /etc/sonic/telemetry/gnmiclient.key " cmd += "-ca_crt /etc/sonic/telemetry/gnmiCA.pem " @@ -364,7 +379,8 @@ def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count): return output['stdout'], output['stderr'] -def gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, interval_ms, count, origin=None, target=None): +def gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, interval_ms, count, origin=None, target=None, + ip=None): """ Send GNMI subscribe request with GNMI client @@ -382,7 +398,7 @@ def gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, interval_ms, co logger.error("path_list is None") return "", "" env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) - ip = duthost.mgmt_ip + ip = ip or duthost.mgmt_ip port = env.gnmi_port cmd = '/root/env-python3/bin/python /root/gnxi/gnmi_cli_py/py_gnmicli.py ' cmd += '--timeout 30 ' @@ -407,7 +423,7 @@ def gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, interval_ms, co return msg, output['stderr'] -def gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, count): +def gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, count, ip=None): """ Send GNMI subscribe request with GNMI client @@ -424,7 +440,7 @@ def gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, count): logger.error("path_list is None") return "", "" env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) - ip = duthost.mgmt_ip + ip = ip or duthost.mgmt_ip port = env.gnmi_port cmd = '/root/env-python3/bin/python /root/gnxi/gnmi_cli_py/py_gnmicli.py ' cmd += '--timeout 120 ' diff --git a/tests/gnmi/test_gnmi_configdb.py b/tests/gnmi/test_gnmi_configdb.py index b051bfbb3f7..8012882ce23 100644 --- a/tests/gnmi/test_gnmi_configdb.py +++ b/tests/gnmi/test_gnmi_configdb.py @@ -5,12 +5,12 @@ import re import time -from .helper import gnmi_set, gnmi_get +from .helper import gnmi_set, gnmi_get, is_mgmt_vrf_enabled from .helper import gnmi_subscribe_polling from .helper import gnmi_subscribe_streaming_sample, gnmi_subscribe_streaming_onchange -from tests.common.helpers.gnmi_utils import add_gnmi_client_common_name +from tests.common.helpers.gnmi_utils import add_gnmi_client_common_name, gnmi_container from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until +from tests.common.utilities import DEFAULT_VRF_NAME, MGMT_VRF_NAME, wait_until from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure logger = logging.getLogger(__name__) @@ -24,7 +24,71 @@ ] -def get_first_interface(duthost): +VRF_SCENARIOS = [ + {"name": "default_1", "vrf": None, "description": "Default (no VRF)"}, + {"name": "default_2", "vrf": DEFAULT_VRF_NAME, "description": f"Default (explicit '{DEFAULT_VRF_NAME}')"}, + {"name": "mgmt", "vrf": MGMT_VRF_NAME, "description": "Management VRF"}, +] + + +@pytest.fixture(scope="module", autouse=True) +def download_gnmi_client(duthosts, rand_one_dut_hostname): + """Stage gnmi_cli on the DUT host filesystem. + + When the gNMI server is bound to a non-default VRF, gnmi_subscribe_polling + must run gnmi_cli via `sudo ip vrf exec /tmp/gnmi_cli ...` on the DUT + host (the gnmi container itself does not have `ip vrf exec` privileges). + The binary ships only inside the container, so copy it out once per module. + """ + duthost = duthosts[rand_one_dut_hostname] + container = gnmi_container(duthost) + duthost.shell("docker cp %s:/usr/sbin/gnmi_cli /tmp/gnmi_cli" % container) + duthost.shell("chmod +x /tmp/gnmi_cli") + + +@pytest.fixture(scope="module", params=VRF_SCENARIOS, ids=lambda scenario: f"vrf_{scenario['name']}") +def vrf_config(request, duthost, ptfhost): + vrf_cfg = request.param.copy() + vrf_cfg.update({ + "dut_ip": duthost.mgmt_ip, + "ptf_ip": ptfhost.mgmt_ip, + "dut_intf": "eth0", + "ptf_intf": "mgmt", + }) + return vrf_cfg + + +@pytest.fixture(scope="module", autouse=True) +def setup_vrf_configuration(duthosts, rand_one_dut_hostname, vrf_config): + """ + This fixture runs before setup_gnmi_server to ensure VRF config is in place. + Only default and mgmt VRFs are supported. + + While these GNMI tests do not depend on SNMP, some tests fail while waiting + for all critical processes to be up and running. These are caused by SNMP + agent address misconfiguration during VRF transition. Hence SNMP services + are stopped before toggling mgmt VRF and restarted after. + """ + duthost = duthosts[rand_one_dut_hostname] + vrf_name = vrf_config["vrf"] + mgmt_vrf_enabled = is_mgmt_vrf_enabled(duthost) + + try: + if vrf_name == MGMT_VRF_NAME and not mgmt_vrf_enabled: + duthost.shell('sudo systemctl stop snmpd snmp-subagent', module_ignore_errors=True) + duthost.shell('sonic-db-cli CONFIG_DB hset "MGMT_VRF_CONFIG|vrf_global" "mgmtVrfEnabled" "true"') + duthost.shell('sudo systemctl start snmpd snmp-subagent', module_ignore_errors=True) + yield vrf_config + + finally: + if vrf_name == MGMT_VRF_NAME and not mgmt_vrf_enabled: + duthost.shell('sudo systemctl stop snmpd snmp-subagent', module_ignore_errors=True) + duthost.shell('sonic-db-cli CONFIG_DB hset "MGMT_VRF_CONFIG|vrf_global" "mgmtVrfEnabled" "false"') + duthost.shell('sonic-db-cli CONFIG_DB hdel "MGMT_VRF_CONFIG|vrf_global" "mgmtVrfEnabled"') + duthost.shell('sudo systemctl start snmpd snmp-subagent', module_ignore_errors=True) + + +def get_first_interface(duthost, excluded_interfaces=[]): if duthost.is_supervisor_node(): return None cmds = "show interface status" @@ -41,8 +105,9 @@ def get_first_interface(duthost): interface_status = line.strip() assert len(interface_status) > 0, "Failed to read interface properties" sl = interface_status.split() + intf_name = sl[0] # Skip portchannel - if sl[lanes_index] == 'N/A': + if sl[lanes_index] == 'N/A' or intf_name in excluded_interfaces: continue if sl[admin_index] == 'up': return sl[0] @@ -78,7 +143,7 @@ def wait_bgp_neighbor(duthost): "Not all BGP sessions are established on DUT") -def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native write, incremental config for configDB Toggle interface admin status @@ -87,7 +152,7 @@ def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost): if duthost.is_supervisor_node(): pytest.skip("gnmi test relies on port data not present on supervisor card '%s'" % rand_one_dut_hostname) file_name = "port.txt" - interface = get_first_interface(duthost) + interface = get_first_interface(duthost, [vrf_config['dut_intf']]) assert interface is not None, "Invalid interface" update_list = ["/sonic-db:CONFIG_DB/localhost/PORT/%s/admin_status:@/root/%s" % (interface, file_name)] path_list = ["/sonic-db:CONFIG_DB/localhost/PORT/%s/admin_status" % (interface)] @@ -97,11 +162,11 @@ def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost): with open(file_name, 'w') as file: file.write(text) ptfhost.copy(src=file_name, dest='/root') - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) # Check interface status and gnmi_get result status = get_interface_status(duthost, "admin_status", interface) assert status == "down", "Incremental config failed to toggle interface %s status" % interface - msg_list = gnmi_get(duthost, ptfhost, path_list) + msg_list = gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) assert msg_list[0] == "\"down\"", msg_list[0] # Startup interface @@ -109,17 +174,17 @@ def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost): with open(file_name, 'w') as file: file.write(text) ptfhost.copy(src=file_name, dest='/root') - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) # Check interface status and gnmi_get result status = get_interface_status(duthost, "admin_status", interface) assert status == "up", "Incremental config failed to toggle interface %s status" % interface - msg_list = gnmi_get(duthost, ptfhost, path_list) + msg_list = gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) assert msg_list[0] == "\"up\"", msg_list[0] # Wait for BGP neighbor to be up wait_bgp_neighbor(duthost) -def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native write, incremental config for configDB GNMI set request with invalid path @@ -134,7 +199,7 @@ def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, ptfhost): file.write(text) ptfhost.copy(src=file_name, dest='/root') try: - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Incremental config failed: " + str(e)) else: @@ -158,7 +223,7 @@ def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, ptfhost): @pytest.mark.parametrize('test_data', test_data_metadata) -def test_gnmi_configdb_polling_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): +def test_gnmi_configdb_polling_01(duthosts, rand_one_dut_hostname, ptfhost, test_data, vrf_config): ''' Verify GNMI subscribe API, streaming onchange mode Subscribe polling mode @@ -166,12 +231,14 @@ def test_gnmi_configdb_polling_01(duthosts, rand_one_dut_hostname, ptfhost, test duthost = duthosts[rand_one_dut_hostname] exp_cnt = 3 path_list = [test_data["path"]] - msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + msg, _ = gnmi_subscribe_polling( + duthost, ptfhost, path_list, 1000, exp_cnt, ip=vrf_config["dut_ip"], vrf_name=vrf_config["vrf"] + ) assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg @pytest.mark.parametrize('test_data', test_data_metadata) -def test_gnmi_configdb_streaming_sample_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): +def test_gnmi_configdb_streaming_sample_01(duthosts, rand_one_dut_hostname, ptfhost, test_data, vrf_config): ''' Verify GNMI subscribe API, streaming onchange mode Subscribe streaming sample mode @@ -180,12 +247,12 @@ def test_gnmi_configdb_streaming_sample_01(duthosts, rand_one_dut_hostname, ptfh exp_cnt = 5 path_list = [test_data["path"]] msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt, - origin="sonic-db") + origin="sonic-db", ip=vrf_config["dut_ip"]) assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg @pytest.mark.parametrize('test_data', test_data_metadata) -def test_gnmi_configdb_streaming_onchange_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): +def test_gnmi_configdb_streaming_onchange_01(duthosts, rand_one_dut_hostname, ptfhost, test_data, vrf_config): ''' Verify GNMI subscribe API, streaming onchange mode Subscribe streaming onchange mode @@ -209,13 +276,13 @@ def worker(duthost, run_flag): client_task.start() exp_cnt = 5 path_list = [test_data["path"]] - msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt*2) + msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt*2, ip=vrf_config["dut_ip"]) run_flag.value = False client_task.join() assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg -def test_gnmi_configdb_streaming_onchange_02(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_streaming_onchange_02(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI subscribe API, streaming onchange mode Subscribe table, and verify gnmi output has table key @@ -236,7 +303,7 @@ def worker(duthost, run_flag): client_task.start() exp_cnt = 3 path_list = ["/sonic-db:CONFIG_DB/localhost/DEVICE_METADATA"] - msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt) + msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt, ip=vrf_config["dut_ip"]) run_flag.value = False client_task.join() @@ -250,7 +317,7 @@ def worker(duthost, run_flag): assert "bgp_asn" in result["localhost"], "Invalid result: " + match -def test_gnmi_configdb_full_replace_01(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_full_replace_01(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native write, full config replace for configDB Toggle interface admin status @@ -258,7 +325,7 @@ def test_gnmi_configdb_full_replace_01(duthosts, rand_one_dut_hostname, ptfhost) duthost = duthosts[rand_one_dut_hostname] if duthost.is_supervisor_node(): pytest.skip("gnmi test relies on port data not present on supervisor card '%s'" % rand_one_dut_hostname) - interface = get_first_interface(duthost) + interface = get_first_interface(duthost, [vrf_config['dut_intf']]) assert interface is not None, "Invalid interface" # Get ASIC namespace and check interface @@ -289,7 +356,7 @@ def check_admin_status(duthost, interface, expected_status): ptfhost.copy(src=filename, dest='/root') replace_list = ["/sonic-db:CONFIG_DB/localhost/:@/root/%s" % filename] - gnmi_set(duthost, ptfhost, [], [], replace_list) + gnmi_set(duthost, ptfhost, [], [], replace_list, ip=vrf_config["dut_ip"]) # Check that interface is down after full config push pytest_assert( @@ -303,7 +370,7 @@ def check_admin_status(duthost, interface, expected_status): wait_bgp_neighbor(duthost) -def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native write with authentication ''' @@ -319,7 +386,7 @@ def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_noaccess" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to set: " + str(e)) assert role in str(e), str(e) @@ -328,7 +395,7 @@ def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_readwrite" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to set: " + str(e)) pytest.fail("Set request failed: " + str(e)) @@ -337,7 +404,7 @@ def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_readonly" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to set: " + str(e)) assert role in str(e), str(e) @@ -346,7 +413,7 @@ def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_set(duthost, ptfhost, [], update_list, []) + gnmi_set(duthost, ptfhost, [], update_list, [], ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to set: " + str(e)) assert "write access" in str(e), str(e) @@ -355,7 +422,7 @@ def test_gnmi_configdb_set_authenticate(duthosts, rand_one_dut_hostname, ptfhost add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") -def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native read with authentication ''' @@ -366,7 +433,7 @@ def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_noaccess" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_get(duthost, ptfhost, path_list) + gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to get: " + str(e)) assert role in str(e), str(e) @@ -375,7 +442,7 @@ def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_readwrite" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_get(duthost, ptfhost, path_list) + gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to get: " + str(e)) pytest.fail("Get request failed: " + str(e)) @@ -384,7 +451,7 @@ def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "gnmi_config_db_readonly" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_get(duthost, ptfhost, path_list) + gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to get: " + str(e)) pytest.fail("Get request failed: " + str(e)) @@ -393,7 +460,7 @@ def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost role = "" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) try: - gnmi_get(duthost, ptfhost, path_list) + gnmi_get(duthost, ptfhost, path_list, ip=vrf_config["dut_ip"]) except Exception as e: logger.info("Failed to get: " + str(e)) pytest.fail("Get request failed: " + str(e)) @@ -402,7 +469,7 @@ def test_gnmi_configdb_get_authenticate(duthosts, rand_one_dut_hostname, ptfhost add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") -def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, ptfhost): +def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, ptfhost, vrf_config): ''' Verify GNMI native read with authentication ''' @@ -413,7 +480,7 @@ def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, p role = "gnmi_config_db_noaccess" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) output, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, 1, - origin="sonic-db") + origin="sonic-db", ip=vrf_config["dut_ip"]) logger.info("GNMI subscribe output: " + output) assert "GRPC error" in output, output assert role in output, output @@ -422,7 +489,7 @@ def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, p role = "gnmi_config_db_readwrite" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) output, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, 1, - origin="sonic-db") + origin="sonic-db", ip=vrf_config["dut_ip"]) assert "GRPC error" not in output, output assert "cloudtype" in output, output @@ -430,7 +497,7 @@ def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, p role = "gnmi_config_db_readonly" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) output, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, 1, - origin="sonic-db") + origin="sonic-db", ip=vrf_config["dut_ip"]) assert "GRPC error" not in output, output assert "cloudtype" in output, output @@ -438,7 +505,7 @@ def test_gnmi_configdb_subscribe_authenticate(duthosts, rand_one_dut_hostname, p role = "" add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic", role) output, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, 1, - origin="sonic-db") + origin="sonic-db", ip=vrf_config["dut_ip"]) assert "GRPC error" not in output, output assert "cloudtype" in output, output diff --git a/tests/gnmi/test_gnoi_system_reboot.py b/tests/gnmi/test_gnoi_system_reboot.py index 95e519b0e78..540cb7f67fc 100644 --- a/tests/gnmi/test_gnoi_system_reboot.py +++ b/tests/gnmi/test_gnoi_system_reboot.py @@ -144,7 +144,7 @@ def test_gnoi_system_reboot_warm(duthosts, rand_one_dut_hostname, localhost, gnm # Wait for critical processes before ending # Warm reboot takes longer for containers to restart; use an extended timeout - wait_critical_processes(duthost, timeout=360) + wait_critical_processes(duthost, timeout=600) # Wait for gNMI container to be running wait_until(120, 10, 0, is_gnmi_container_running, duthost) diff --git a/tests/ha/configs/privatelink_config.py b/tests/ha/configs/privatelink_config.py index 3eed8a36fa3..7ff806c7491 100644 --- a/tests/ha/configs/privatelink_config.py +++ b/tests/ha/configs/privatelink_config.py @@ -20,6 +20,10 @@ PL_ENCODING_MASK = "::ffff:ffff:ffff:0:0" PL_OVERLAY_SIP = "fd41:108:20:abc:abc::0" PL_OVERLAY_SIP_MASK = "ffff:ffff:ffff:ffff:ffff:ffff::" +PL_ENCODING_IP_ALTERNATE = "fd40::d107:64:ff71:0:0" +PL_ENCODING_MASK_ALTERNATE = "fffe:0:0:ffff:ffff:ffff::" +PL_OVERLAY_SIP_ALTERNATE = "1:108:20::" +PL_OVERLAY_SIP_MASK_ALTERNATE = "1:ffff:ffff::" PL_OVERLAY_DIP = "2603:10e1:100:2::3401:203" PL_OVERLAY_DIP_MASK = "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" PL_REDIRECT_OVERLAY_DIP = "2603:10e1:100:2::0" @@ -453,3 +457,12 @@ "port_map": PORT_MAP_1, } } + +DEFAULT_PL_SIP = (PL_ENCODING_IP, PL_ENCODING_MASK, PL_OVERLAY_SIP, PL_OVERLAY_SIP_MASK) +PL_SIP_ALTERNATE = ( + PL_ENCODING_IP_ALTERNATE, + PL_ENCODING_MASK_ALTERNATE, + PL_OVERLAY_SIP_ALTERNATE, + PL_OVERLAY_SIP_MASK_ALTERNATE, +) +PL_SIP_CONFIGS = (ENI_CONFIG, PE_VNET_MAPPING_CONFIG) diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index 9bbd36509a8..4f1929f9774 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -5,6 +5,7 @@ from pathlib import Path from collections import defaultdict import os +from tests.conftest import get_specified_dpus from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.ha.smartswitch_ha_helper import PtfTcpTestAdapter @@ -22,14 +23,16 @@ LOCAL_DUT_INTF, REMOTE_DUT_INTF, \ REMOTE_PTF_SEND_INTF, REMOTE_PTF_RECV_INTF, VXLAN_UDP_BASE_SRC_PORT, VXLAN_UDP_SRC_PORT_MASK, \ NPU_DATAPLANE_IP, NPU_DATAPLANE_MAC, NPU_DATAPLANE_PORT, DPU_DATAPLANE_IP, DPU_DATAPLANE_MAC, DPU_DATAPLANE_PORT -from tests.common.dash_utils import render_template_to_host, apply_swssconfig_file +from tests.common.dash_utils import render_template_to_host, apply_swssconfig_file, apply_dash_configs from tests.common.helpers.smartswitch_util import correlate_dpu_info_with_dpuhost, get_data_port_on_dpu, get_dpu_dataplane_port # noqa F401 from tests.ha.gnmi_utils import generate_gnmi_cert, apply_gnmi_cert, recover_gnmi_cert, apply_gnmi_file, apply_messages from tests.ha.ha_gnmi import apply_ha_messages, ha_scope_config, ha_set_config -from tests.common import config_reload import configs.privatelink_config as pl from tests.common.helpers.assertions import pytest_require as pt_require +from tests.common.helpers.assertions import pytest_assert as pt_assert +from tests.common.utilities import wait_until from tests.ha.ha_utils import ( + parallel_config_reload_dpuhosts, wait_for_pending_operation_id, verify_ha_state, set_dash_ha_scope @@ -39,10 +42,87 @@ logger = logging.getLogger(__name__) +def _get_dpu_neighbor_ip(role_index, dpu_index): + return f"20.0.{200 + role_index}.{dpu_index + 1}" + + +def _get_appl_db_neighbor_key(duthost, neighbor_ip): + result = duthost.shell( + f'sonic-db-cli APPL_DB KEYS "NEIGH_TABLE:*:{neighbor_ip}"', + module_ignore_errors=True, + ) + if result.get("rc", 0) != 0: + logger.debug( + "%s failed querying APPL_DB for %s: rc=%s stderr=%s", + duthost.hostname, + neighbor_ip, + result.get("rc"), + result.get("stderr", "").strip(), + ) + return None + + return next( + (key for key in result.get("stdout_lines", []) if key.endswith(neighbor_ip)), + None, + ) + + +def wait_for_dpu_neighbor_resolution(duthost, role_index, dpu_index, timeout=120, interval=2): + neighbor_ip = _get_dpu_neighbor_ip(role_index, dpu_index) + + def _neighbor_is_resolved(): + neighbor_key = _get_appl_db_neighbor_key(duthost, neighbor_ip) + if not neighbor_key: + return False + + neigh_result = duthost.shell( + f'sonic-db-cli APPL_DB HGET "{neighbor_key}" neigh', + module_ignore_errors=True, + ) + if neigh_result.get("rc", 0) != 0: + return False + + neighbor_mac = neigh_result["stdout"].strip() + if not neighbor_mac: + return False + + logger.info( + "%s resolved DPU neighbor %s in %s with MAC %s", + duthost.hostname, + neighbor_ip, + neighbor_key, + neighbor_mac, + ) + return True + + logger.info( + "Waiting for DPU%s neighbor %s to resolve in APPL_DB on %s", + dpu_index, + neighbor_ip, + duthost.hostname, + ) + pt_assert( + wait_until(timeout, interval, 0, _neighbor_is_resolved), + f"Timed out waiting for APPL_DB NEIGH_TABLE entry for {neighbor_ip} on {duthost.hostname}", + ) + + @pytest.fixture(scope="session") -def dpuhosts(dpuhosts): - """Limit to the first 2 DPU hosts for all HA tests.""" - return dpuhosts.nodes[:2] +def dpuhosts(all_dpuhosts, request): + """Limit standard HA tests to the first 2 requested DPU hosts.""" + requested_dpuhosts = get_specified_dpus(request) + if not requested_dpuhosts: + return all_dpuhosts.nodes[:2] + + nodes_by_hostname = {node.hostname: node for node in all_dpuhosts.nodes} + missing_dpuhosts = [ + hostname for hostname in requested_dpuhosts if hostname not in nodes_by_hostname + ] + pt_require( + not missing_dpuhosts, + f"Requested DPU hosts were not initialized: {missing_dpuhosts}", + ) + return [nodes_by_hostname[hostname] for hostname in requested_dpuhosts[:2]] ha_scope_per_dut = [ @@ -393,17 +473,13 @@ def vxlan_udp_dport(request, duthost): config_vxlan_udp_dport(duthost, 4789) -@pytest.fixture(scope="module") -def set_vxlan_udp_sport_range(dpuhosts): - """ - Configure VXLAN UDP source port range in dpu configuration. - - """ +def _apply_vxlan_udp_sport_range(dpuhosts): vxlan_sport_config = [ { "SWITCH_TABLE:switch": { "vxlan_sport": VXLAN_UDP_BASE_SRC_PORT, - "vxlan_mask": VXLAN_UDP_SRC_PORT_MASK + "vxlan_mask": VXLAN_UDP_SRC_PORT_MASK, + "vxlan_port": "4789" }, "OP": "SET" } @@ -414,14 +490,36 @@ def set_vxlan_udp_sport_range(dpuhosts): for dpuhost in dpuhosts: dpuhost.copy(content=json.dumps(vxlan_sport_config, indent=4), dest=config_path, verbose=False) apply_swssconfig_file(dpuhost, config_path) - if 'pensando' in dpuhost.facts['asic_type']: - logger.warning("Applying Pensando DPU VXLAN sport workaround") - dpuhost.shell("pdsctl debug update device --vxlan-port 4789 --vxlan-src-ports 5120-5247") + + +@pytest.fixture(scope="module") +def set_vxlan_udp_sport_range(dpuhosts): + """ + Configure VXLAN UDP source port range in dpu configuration. + + """ + _apply_vxlan_udp_sport_range(dpuhosts) yield + dpuhosts_to_reload = [] for dpuhost in dpuhosts: if str(VXLAN_UDP_BASE_SRC_PORT) in dpuhost.shell("redis-cli -n 0" " hget SWITCH_TABLE:switch vxlan_sport")['stdout']: - config_reload(dpuhost, safe_reload=True, yang_validate=False) + dpuhosts_to_reload.append(dpuhost) + parallel_config_reload_dpuhosts(dpuhosts_to_reload) + + +@pytest.fixture(scope="function") +def ensure_vxlan_udp_sport_range(set_vxlan_udp_sport_range, dpuhosts): + dpuhosts_to_configure = [] + for dpuhost in dpuhosts: + vxlan_sport = dpuhost.shell("redis-cli -n 0" + " hget SWITCH_TABLE:switch vxlan_sport")['stdout'] + if str(VXLAN_UDP_BASE_SRC_PORT) not in vxlan_sport: + dpuhosts_to_configure.append(dpuhost) + + if dpuhosts_to_configure: + logger.info("Re-applying VXLAN source port config after per-test cleanup reset") + _apply_vxlan_udp_sport_range(dpuhosts_to_configure) @pytest.fixture(scope="module") @@ -429,6 +527,31 @@ def dpu_index(request): return request.config.getoption("--dpu_index") +def _apply_pl_sip(sip_params): + encoding_ip, encoding_mask, overlay_sip, overlay_sip_mask = sip_params + pl.PL_ENCODING_IP = encoding_ip + pl.PL_ENCODING_MASK = encoding_mask + pl.PL_OVERLAY_SIP = overlay_sip + pl.PL_OVERLAY_SIP_MASK = overlay_sip_mask + pl_sip_encoding = f"{encoding_ip}/{encoding_mask}" + overlay_sip_prefix = f"{overlay_sip}/{overlay_sip_mask}" + for cfg in pl.PL_SIP_CONFIGS: + for entry in cfg.values(): + if "pl_sip_encoding" in entry: + entry["pl_sip_encoding"] = pl_sip_encoding + if "overlay_sip_prefix" in entry: + entry["overlay_sip_prefix"] = overlay_sip_prefix + + +@pytest.fixture(scope="function", autouse=True) +def configure_pl_sip_for_platform(request): + if "dpuhosts" not in request.fixturenames: + return + dpuhost = request.getfixturevalue("dpuhosts")[request.getfixturevalue("dpu_index")] + sip_params = pl.PL_SIP_ALTERNATE if "bluefield" in dpuhost.facts["asic_type"] else pl.DEFAULT_PL_SIP + _apply_pl_sip(sip_params) + + @pytest.fixture(scope="module") def dpu_setup(duthosts, dpuhosts, dpu_index, skip_config): if skip_config: @@ -545,8 +668,7 @@ def setup_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_g logger.info("HA: setup from json for Primary and Standby") - # Workaround for the neigh resolve issue - # To be removed after fixes are merged: PR 147, 148 in sonic-dash-ha + # TODO: remove once neighbor flakiness is fixed. for i in range(len(duthosts)): logger.info(f"Sending ping to DPU{dpuhosts[i].dpu_index} for {duthosts[i].hostname}") ip_part = 200 + i @@ -554,6 +676,13 @@ def setup_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_g ping_result = duthosts[i].shell(f"ping -c 3 20.0.{ip_part}.{ip_last}", module_ignore_errors=True)["stdout"] logger.info(f"{duthosts[i].hostname} ping_result [{ping_result}]") + for i in range(len(duthosts)): + wait_for_dpu_neighbor_resolution( + duthost=duthosts[i], + role_index=i, + dpu_index=dpuhosts[i].dpu_index, + ) + with open(ha_set_file) as f: ha_set_data = json.load(f)["DASH_HA_SET_CONFIG_TABLE"] @@ -773,10 +902,12 @@ def activate_dash_ha_from_json(duthosts, dpuhosts, localhost, ptfhost, setup_gnm deactivate_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) -def apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost): +def apply_dash_pl_pipeline_config( + localhost, duthosts, dpuhosts, ptfhost, floating_nic=False, set_db=True, wait_after_apply=5 +): """ Apply DASH Private Link pipeline config (appliance, routing type, VNET, - ENI, routes, meters) on all DPUs. Required by any test that sends PL + ENI/FNIC, routes, meters) on all DPUs. Required by any test that sends PL traffic and does not already pull in the steady-state common_setup_teardown. """ @@ -784,42 +915,67 @@ def apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost): duthost = duthosts[i] dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - } logger.info( - f"setup_dash_pl_pipeline: applying base config on " + f"setup_dash_pl_pipeline: applying DASH PL config on " f"{duthost.hostname} dpu {dpuhost.dpu_index}" ) - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG, - } - if "bluefield" in dpuhost.facts["asic_type"]: - route_and_mapping_messages.update({**pl.INBOUND_VNI_ROUTE_RULE_CONFIG}) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + if floating_nic: + apply_dash_configs( + localhost, + duthost, + ptfhost, + dpuhost.dpu_index, + pl.APPLIANCE_FNIC_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.ROUTING_TYPE_VNET_CONFIG, + pl.VNET_CONFIG, + pl.METER_POLICY_V4_CONFIG, + pl.TUNNEL1_CONFIG, + pl.METER_RULE1_V4_CONFIG, + pl.METER_RULE2_V4_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.PE_VNET_MAPPING_CONFIG, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_VNET_MAPPING_CONFIG, + pl.VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT, + pl.VM_VNI_ROUTE_RULE_CONFIG if "bluefield" in dpuhost.facts["asic_type"] else None, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG if "bluefield" in dpuhost.facts["asic_type"] else None, + pl.TRUSTED_VNI_ROUTE_RULE_CONFIG if "bluefield" in dpuhost.facts["asic_type"] else None, + pl.ENI_FNIC_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + set_db=set_db, + wait_after_apply=wait_after_apply, + apply_fn=apply_messages, + ) + else: + apply_dash_configs( + localhost, + duthost, + ptfhost, + dpuhost.dpu_index, + pl.APPLIANCE_CONFIG, + pl.ROUTING_TYPE_PL_CONFIG, + pl.VNET_CONFIG, + pl.METER_POLICY_V4_CONFIG, + pl.METER_RULE1_V4_CONFIG, + pl.METER_RULE2_V4_CONFIG, + pl.ROUTE_GROUP1_CONFIG, + pl.PE_VNET_MAPPING_CONFIG, + pl.PE_SUBNET_ROUTE_CONFIG, + pl.VM_SUBNET_ROUTE_CONFIG, + pl.INBOUND_VNI_ROUTE_RULE_CONFIG if "bluefield" in dpuhost.facts["asic_type"] else None, + pl.ENI_CONFIG, + pl.ENI_ROUTE_GROUP1_CONFIG, + set_db=set_db, + wait_after_apply=wait_after_apply, + apply_fn=apply_messages, + ) @pytest.fixture(scope="function") def setup_dash_pl_pipeline( - localhost, duthosts, ptfhost, dpu_index, skip_config, - dpuhosts, setup_npu_dpu, set_vxlan_udp_sport_range + localhost, duthosts, ptfhost, dpu_index, skip_config, + dpuhosts, setup_npu_dpu, ensure_vxlan_udp_sport_range ): if skip_config: yield @@ -827,8 +983,7 @@ def setup_dash_pl_pipeline( apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield logger.info("setup_dash_pl_pipeline: cleanup.") - for dpuhost in dpuhosts: - config_reload(dpuhost, safe_reload=True, yang_validate=False) + parallel_config_reload_dpuhosts(dpuhosts) @pytest.fixture(scope="module") @@ -842,5 +997,4 @@ def setup_dash_pl_pipeline_module_scope( apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield logger.info("setup_dash_pl_pipeline: cleanup.") - for dpuhost in dpuhosts: - config_reload(dpuhost, safe_reload=True, yang_validate=False) + parallel_config_reload_dpuhosts(dpuhosts) diff --git a/tests/ha/gnmi_utils.py b/tests/ha/gnmi_utils.py index 152810813ff..bb8236439be 100644 --- a/tests/ha/gnmi_utils.py +++ b/tests/ha/gnmi_utils.py @@ -205,29 +205,6 @@ def apply_gnmi_cert(my_duthost, ptfhost): ptfhost.copy(src=env.work_dir+env.gnmi_client_cert, dest=my_dest) ptfhost.copy(src=env.work_dir+env.gnmi_client_key, dest=my_dest) _set_gnmi_client_cert_role(my_duthost) - certs_table = "{}|certs".format(env.gnmi_config_table) - my_duthost.shell( - 'sudo sonic-db-cli CONFIG_DB hset "{}" ca_crt "{}{}"'.format( - certs_table, env.gnmi_cert_path, env.gnmi_ca_cert - ), - module_ignore_errors=True, - ) - my_duthost.shell( - 'sudo sonic-db-cli CONFIG_DB hset "{}" server_crt "{}{}"'.format( - certs_table, env.gnmi_cert_path, env.gnmi_server_cert - ), - module_ignore_errors=True, - ) - my_duthost.shell( - 'sudo sonic-db-cli CONFIG_DB hset "{}" server_key "{}{}"'.format( - certs_table, env.gnmi_cert_path, env.gnmi_server_key - ), - module_ignore_errors=True, - ) - my_duthost.shell( - 'sudo sonic-db-cli CONFIG_DB hset "{}|gnmi" client_auth true'.format(env.gnmi_config_table), - module_ignore_errors=True, - ) port = env.gnmi_port assert int(port) > 0, "Invalid GNMI port" dut_command = "docker exec %s supervisorctl stop %s" % (env.gnmi_container, env.gnmi_program) diff --git a/tests/ha/ha_bgp_utils.py b/tests/ha/ha_bgp_utils.py index 3476b7ab881..0c4dae20d47 100644 --- a/tests/ha/ha_bgp_utils.py +++ b/tests/ha/ha_bgp_utils.py @@ -1,9 +1,61 @@ +import json import logging +from tests.common.helpers.assertions import pytest_assert + logger = logging.getLogger(__name__) +def _get_t2_peer_ips(duthost): + """Return DUT-side BGP peer IPs whose remote description identifies them as T2.""" + out = duthost.shell('vtysh -c "show ip bgp summary json"')["stdout"] + summary = json.loads(out) + peers = summary.get("ipv4Unicast", {}).get("peers", {}) + return [ + ip for ip, p in peers.items() + if "T2" in (p.get("hostname") or "") or "T2" in (p.get("desc") or "") + ] + + +def check_vip_advertised_to_t2(duthosts, vip): + """Verify the DUT advertises the VIP /32 prefix to every T2 BGP peer.""" + vip_prefix = "{}/32".format(vip) + # DutHosts proxies any attribute name to its node list, so hasattr() lies. + # A real single host has a string `hostname`; DutHosts.hostname is a method. + if isinstance(getattr(duthosts, "hostname", None), str): + duts = [duthosts] + else: + duts = list(duthosts) + + missing = [] + found_any = False + for duthost in duts: + t2_peer_ips = _get_t2_peer_ips(duthost) + if not t2_peer_ips: + logger.info("%s: no T2 BGP peers found", duthost.hostname) + continue + for peer_ip in t2_peer_ips: + cmd = 'vtysh -c "show ip bgp neighbors {} advertised-routes json"'.format(peer_ip) + res = duthost.shell(cmd, module_ignore_errors=True) + adv = {} + if not res.get("failed"): + try: + adv = json.loads(res["stdout"]).get("advertisedRoutes", {}) + except ValueError: + adv = {} + if vip_prefix in adv: + found_any = True + logger.info("%s -> %s: VIP %s advertised", duthost.hostname, peer_ip, vip_prefix) + else: + missing.append("{}->{}".format(duthost.hostname, peer_ip)) + + pytest_assert(found_any and not missing, + "VIP {} not advertised to T2 peers: {}".format(vip_prefix, missing)) + logger.info("VIP %s advertised to all T2 peers on %s", + vip_prefix, [d.hostname for d in duts]) + + def _ha_bgp_oper(duthost, start=True): cmd = 'show ip bgp summary' diff --git a/tests/ha/ha_link_utils.py b/tests/ha/ha_link_utils.py index 1fb08393cab..d1cee44a94c 100644 --- a/tests/ha/ha_link_utils.py +++ b/tests/ha/ha_link_utils.py @@ -8,6 +8,30 @@ ACL_RULE = "DROP_ALL" ACL_DIR = "egress" +DPU_DATAPLANE_PORT = "Ethernet0" + + +def shutdown_dpu_dataplane_port(dpuhost, interface=DPU_DATAPLANE_PORT): + """ + Simulate a DPU dataplane link going down by administratively shutting + down the DPU's dataplane Ethernet interface. + """ + logger.info( + f"{dpuhost.hostname} Shutting down DPU dataplane interface {interface}" + ) + dpuhost.shell(f"config interface shutdown {interface}") + + +def startup_dpu_dataplane_port(dpuhost, interface=DPU_DATAPLANE_PORT): + """ + Restore the DPU dataplane link by administratively starting up the DPU's + dataplane Ethernet interface. + """ + logger.info( + f"{dpuhost.hostname} Starting up DPU dataplane interface {interface}" + ) + dpuhost.shell(f"config interface startup {interface}") + def add_acl_link_drop(duthost, interface): """ diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py index ee1cd2f4704..c462da9d688 100644 --- a/tests/ha/ha_utils.py +++ b/tests/ha/ha_utils.py @@ -1,13 +1,31 @@ import logging import json import os +from concurrent.futures import ThreadPoolExecutor +import configs.privatelink_config as pl +from tests.common.config_reload import config_reload from tests.common.utilities import wait_until from tests.ha.ha_gnmi import apply_ha_messages, ha_scope_config, ha_set_config +from gnmi_utils import apply_messages logger = logging.getLogger(__name__) +def _config_reload_dpuhost(dpuhost): + logger.info(f"config reload on {dpuhost.hostname}") + config_reload(dpuhost, safe_reload=True, yang_validate=False) + + +def parallel_config_reload_dpuhosts(dpuhosts): + dpuhosts = list(dpuhosts) + if not dpuhosts: + return + + with ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: + list(executor.map(_config_reload_dpuhost, dpuhosts)) + + def build_dash_ha_scope_args(fields): """ Build args for DASH_HA_SCOPE_CONFIG_TABLE @@ -172,14 +190,16 @@ def verify_ha_state( expected_state, timeout=120, interval=5, + ack=True ): """ Wait until HA reaches the expected state by querying STATE_DB. """ def _check_ha_state(): db_key = "DASH_HA_SCOPE_STATE|" + scope_key.replace(":", "|") + var = "local_acked_asic_ha_state" if ack else "local_ha_state" res = duthost.shell( - f'sonic-db-cli STATE_DB HGET "{db_key}" local_acked_asic_ha_state' + f'sonic-db-cli STATE_DB HGET "{db_key}" {var}' ) state = res["stdout"].strip() return state == expected_state @@ -434,3 +454,42 @@ def bfd_unpin_both_sides(localhost, ptfhost, duthosts): ptfhost=ptfhost, messages=ha_set_messages, ) + + +def program_eni_pl_on_dpu(localhost, ptfhost, duthost, dpuhost): + """ + Apply the full DASH PL pipeline configuration (appliance, routing types, + VNET, routes, meters and ENI) on the DPU . + """ + + base_config_messages = { + **pl.APPLIANCE_CONFIG, + **pl.ROUTING_TYPE_PL_CONFIG, + **pl.VNET_CONFIG, + **pl.ROUTE_GROUP1_CONFIG, + **pl.METER_POLICY_V4_CONFIG + } + logger.info( + f"HA: Programming ENI PL on DPU: " + f"{duthost.hostname} dpu {dpuhost.dpu_index}" + ) + apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) + + route_and_mapping_messages = { + **pl.PE_VNET_MAPPING_CONFIG, + **pl.PE_SUBNET_ROUTE_CONFIG, + **pl.VM_SUBNET_ROUTE_CONFIG + } + if 'bluefield' in dpuhost.facts['asic_type']: + route_and_mapping_messages.update({**pl.INBOUND_VNI_ROUTE_RULE_CONFIG}) + apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) + + meter_rule_messages = { + **pl.METER_RULE1_V4_CONFIG, + **pl.METER_RULE2_V4_CONFIG, + } + apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) + + apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) + apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + logger.info(f"HA: ENI programming on {dpuhost.hostname} completed") diff --git a/tests/ha/test_ha_bfd_pin.py b/tests/ha/test_ha_bfd_pin.py index d05f5c1975e..3cd08c5884d 100644 --- a/tests/ha/test_ha_bfd_pin.py +++ b/tests/ha/test_ha_bfd_pin.py @@ -1,6 +1,5 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time @@ -10,12 +9,17 @@ LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from tests.common.helpers.assertions import pytest_assert from ha_dash_flow_utils import compare_flow_tables_pdsctl -from ha_utils import bfd_pin_primary, bfd_unpin_primary, bfd_pin_both_sides, bfd_unpin_both_sides +from ha_utils import ( + bfd_pin_primary, + bfd_unpin_primary, + bfd_pin_both_sides, + bfd_unpin_both_sides, + parallel_config_reload_dpuhosts, +) logger = logging.getLogger(__name__) @@ -38,58 +42,16 @@ def common_setup_teardown( setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server, - set_vxlan_udp_sport_range, + ensure_vxlan_udp_sport_range, setup_npu_dpu # noqa: F811 ): if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Starting DASH configuration on {duthost.hostname} dpu {dpuhost.dpu_index} " - f"with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - for dpuhost in dpuhosts: - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) + parallel_config_reload_dpuhosts(dpuhosts) """ diff --git a/tests/ha/test_ha_bgp_down.py b/tests/ha/test_ha_bgp_down.py index 8cecd647bc7..988d652889f 100644 --- a/tests/ha/test_ha_bgp_down.py +++ b/tests/ha/test_ha_bgp_down.py @@ -1,6 +1,5 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time @@ -10,12 +9,12 @@ LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from tests.common.helpers.assertions import pytest_assert -from ha_dash_flow_utils import compare_flow_tables_pdsctl +from ha_dash_flow_utils import compare_flow_tables from ha_bgp_utils import ha_bgp_shutdown, ha_bgp_start +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -24,7 +23,7 @@ pytest.mark.skip_check_dut_health ] -TRAFFIC_LOSS_THRESHOLD_PERCENTAGE = 2.0 +TRAFFIC_LOSS_TIME_THRESHOLD = 2 # seconds @pytest.fixture(autouse=True, scope="function") @@ -38,58 +37,16 @@ def common_setup_teardown( setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server, - set_vxlan_udp_sport_range, + ensure_vxlan_udp_sport_range, setup_npu_dpu # noqa: F811 ): if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Starting DASH configuration on {duthost.hostname} dpu {dpuhost.dpu_index} " - f"with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - for dpuhost in dpuhosts: - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) + parallel_config_reload_dpuhosts(dpuhosts) """ @@ -105,11 +62,11 @@ def common_setup_teardown( @pytest.mark.parametrize( "bgp_shut_on_standby", [True, False], - ids=["Standby BGP Shut", "Primary BGP Shut"] + ids=["Standby_BGP_Shut", "Primary_BGP_Shut"] ) @pytest.mark.parametrize( "traffic_to_standby", [True, False], - ids=["Standby Traffic", "Primary Traffic"] + ids=["Standby_Traffic", "Primary_Traffic"] ) def test_ha_bgp_shut( localhost, @@ -169,7 +126,7 @@ def bgp_shut_ha_action(): testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) if send_count == 0: logger.info("First packet verified on standby - compare flows") - flow_op = compare_flow_tables_pdsctl(dpuhosts[0], dpuhosts[1]) + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) pytest_assert(flow_op, "Expected identical flow tables on primary and standby") else: @@ -179,7 +136,7 @@ def bgp_shut_ha_action(): testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) if send_count == 0: logger.info("First packet verified on primary - compare flows") - flow_op = compare_flow_tables_pdsctl(dpuhosts[0], dpuhosts[1]) + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) pytest_assert(flow_op, "Expected identical flow tables on primary and standby") except Exception as e: if failed_count == 0: @@ -197,16 +154,16 @@ def bgp_shut_ha_action(): time.sleep(2) traffic = "traffic to standby" if traffic_to_standby else "traffic to primary" bgp_shut = "Standby BGP shut" if traffic_to_standby else "Primary BGP shut" - threshold_loss = TRAFFIC_LOSS_THRESHOLD_PERCENTAGE + threshold_loss = rate_pps * TRAFFIC_LOSS_TIME_THRESHOLD percentage_loss = (failed_count / send_count) * 100 if bgp_shut_on_standby: ha_bgp_start(duthosts[1]) else: ha_bgp_start(duthosts[0]) - if (percentage_loss < threshold_loss): - logger.info(f"{bgp_shut} with {traffic} test passed. Sent: {send_count}," + if (failed_count < threshold_loss): + logger.info(f"{bgp_shut} with {traffic} test passed. Sent: {send_count}, " f" lost: {failed_count}, percentage loss: {percentage_loss}, threshold: {threshold_loss}") else: - pytest.fail(f"{bgp_shut} with {traffic} test failed. Sent: {send_count}," + pytest.fail(f"{bgp_shut} with {traffic} test failed. Sent: {send_count}, " f" lost: {failed_count} percentage loss: {percentage_loss}, threshold: {threshold_loss}") diff --git a/tests/ha/test_ha_config_reload.py b/tests/ha/test_ha_config_reload.py index c76e5c56791..1ef24bca483 100644 --- a/tests/ha/test_ha_config_reload.py +++ b/tests/ha/test_ha_config_reload.py @@ -1,6 +1,5 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time @@ -10,11 +9,12 @@ LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from tests.common.helpers.assertions import pytest_assert from ha_dash_flow_utils import compare_flow_tables_pdsctl +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -37,58 +37,16 @@ def common_setup_teardown( setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server, - set_vxlan_udp_sport_range, + ensure_vxlan_udp_sport_range, setup_npu_dpu # noqa: F811 ): if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Starting DASH configuration on {duthost.hostname} dpu {dpuhost.dpu_index} " - f"with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - for dpuhost in dpuhosts: - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) + parallel_config_reload_dpuhosts(dpuhosts) """ diff --git a/tests/ha/test_ha_dpu_power_down.py b/tests/ha/test_ha_dpu_power_down.py index c7d1584feae..e13837db5ac 100644 --- a/tests/ha/test_ha_dpu_power_down.py +++ b/tests/ha/test_ha_dpu_power_down.py @@ -1,21 +1,19 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time import threading -import concurrent.futures from constants import ( LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets from tests.common.helpers.assertions import pytest_assert -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from ha_dash_flow_utils import compare_flow_tables from ha_dpu_utils import dpu_power_off_for_index, dpu_power_on_for_index +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -30,11 +28,6 @@ INITIAL_SEND_COUNT = 100 -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - @pytest.fixture(autouse=True, scope="function") def common_setup_teardown( localhost, @@ -46,58 +39,16 @@ def common_setup_teardown( setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server, - set_vxlan_udp_sport_range, + ensure_vxlan_udp_sport_range, setup_npu_dpu # noqa: F811 ): if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Starting DASH configuration on {duthost.hostname}" - "dpu {dpuhost.dpu_index} with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - # Map the reload_config_for_host function to the dpuhosts list - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) """ diff --git a/tests/ha/test_ha_dpu_process_crash.py b/tests/ha/test_ha_dpu_process_crash.py index 44c75ba7405..0bfad58cefc 100644 --- a/tests/ha/test_ha_dpu_process_crash.py +++ b/tests/ha/test_ha_dpu_process_crash.py @@ -47,6 +47,7 @@ DPU_CRITICAL_PROCESSES = [ pytest.param("syncd", "syncd", id="syncd"), pytest.param("bgpd", "bgp", id="bgp"), + pytest.param("orchagent", "swss", id="swss"), ] @@ -77,6 +78,7 @@ def verify_ha_state_converged(duthost, scope_key, expected_state): expected_state=expected_state, timeout=HA_CONVERGENCE_TIMEOUT, interval=HA_CHECK_INTERVAL, + ack=False ), ( f"{duthost.hostname}: HA scope '{scope_key}' did not reach " f"'{expected_state}' within {HA_CONVERGENCE_TIMEOUT}s" @@ -144,6 +146,11 @@ def standby_dpuhost(dpuhosts): class TestDpuProcessCrash: + @pytest.fixture(autouse=True) + def _setup(self, ha_owner): + self.expected_ha_state_after_crash = "active" if ha_owner == "dpu" else "HA_STATE_STANDBY" + self.expected_ha_state_verify = "active" if ha_owner == "dpu" else "HA_STATE_STANDALONE" + def _run( self, process_name, container, crash_dpuhost, crash_duthost, crash_scope_key, @@ -225,10 +232,10 @@ def test_crash_active_dpu_traffic_on_active( process_name=process_name, container=container, crash_dpuhost=primary_dpuhost, crash_duthost=primary_dut, crash_scope_key=primary_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash=self.expected_ha_state_after_crash, verify_duthost=standby_dut, verify_scope_key=standby_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify=self.expected_ha_state_verify, ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=0, ) @@ -245,10 +252,10 @@ def test_crash_active_dpu_traffic_on_standby( process_name=process_name, container=container, crash_dpuhost=primary_dpuhost, crash_duthost=primary_dut, crash_scope_key=primary_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash=self.expected_ha_state_after_crash, verify_duthost=standby_dut, verify_scope_key=standby_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify=self.expected_ha_state_verify, ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=1, ) @@ -265,10 +272,10 @@ def test_crash_standby_dpu_traffic_on_active( process_name=process_name, container=container, crash_dpuhost=standby_dpuhost, crash_duthost=standby_dut, crash_scope_key=standby_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash=self.expected_ha_state_after_crash, verify_duthost=primary_dut, verify_scope_key=primary_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify=self.expected_ha_state_verify, ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=0, ) @@ -285,10 +292,10 @@ def test_crash_standby_dpu_traffic_on_standby( process_name=process_name, container=container, crash_dpuhost=standby_dpuhost, crash_duthost=standby_dut, crash_scope_key=standby_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash=self.expected_ha_state_after_crash, verify_duthost=primary_dut, verify_scope_key=primary_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify=self.expected_ha_state_verify, ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=1, ) diff --git a/tests/ha/test_ha_eni_out_of_order.py b/tests/ha/test_ha_eni_out_of_order.py index e36cb0fa86e..48df964d6fe 100644 --- a/tests/ha/test_ha_eni_out_of_order.py +++ b/tests/ha/test_ha_eni_out_of_order.py @@ -1,9 +1,7 @@ import logging -import concurrent.futures import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.config_reload import config_reload from tests.ha.conftest import ( apply_dash_pl_pipeline_config, setup_dash_ha_from_json_util, @@ -11,7 +9,7 @@ activate_dash_ha_from_json_util, deactivate_dash_ha_from_json_util ) -from ha_utils import verify_ha_state +from ha_utils import parallel_config_reload_dpuhosts, verify_ha_state logger = logging.getLogger(__name__) @@ -20,11 +18,6 @@ ] -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - def test_ha_eni_out_of_order( ptfadapter, localhost, @@ -57,8 +50,7 @@ def test_ha_eni_out_of_order( deactivate_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) remove_setup_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) # cleanup the ENI - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) logger.info("HA: reprogram HA and ENI") setup_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) @@ -71,6 +63,5 @@ def test_ha_eni_out_of_order( "Standby HA state is not active") finally: deactivate_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) remove_setup_dash_ha_from_json_util(duthosts, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner) diff --git a/tests/ha/test_ha_launch_with_no_peer.py b/tests/ha/test_ha_launch_with_no_peer.py index cc7fc2b8142..45e01a39a3d 100644 --- a/tests/ha/test_ha_launch_with_no_peer.py +++ b/tests/ha/test_ha_launch_with_no_peer.py @@ -2,18 +2,26 @@ import json import pytest import logging +import ptf.testutils as testutils +from constants import ( + LOCAL_PTF_INTF, + REMOTE_PTF_RECV_INTF, +) from conftest import ( activate_scope_per_dut, deactivate_dash_ha_from_json_util, ha_scope_per_dut, - remove_setup_dash_ha_from_json_util + remove_setup_dash_ha_from_json_util, + wait_for_dpu_neighbor_resolution, ) +from packets import outbound_pl_packets from ha_utils import verify_ha_state, wait_for_pending_operation_id, ha_scope_config, ha_set_config, apply_ha_messages from tests.common.helpers.assertions import pytest_assert pytestmark = [ - pytest.mark.topology("t1-smartswitch-ha") + pytest.mark.topology("t1-smartswitch-ha"), + pytest.mark.skip_check_dut_health, ] logger = logging.getLogger(__name__) @@ -39,7 +47,7 @@ def setup_dash_ha(duthost, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_o scope_fields = dict(scope_fields) scope_fields['owner'] = ha_owner - # Workaround for the neigh resolve issue + # TODO: remove once neighbor flakiness is fixed. ''' ip_part = 200 + role_index ip_last = dpuhost.dpu_index + 1 @@ -48,6 +56,12 @@ def setup_dash_ha(duthost, dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_o logger.info(f"{duthost.hostname} ping_result [{ping_result}]") ''' + wait_for_dpu_neighbor_resolution( + duthost=duthost, + role_index=role_index, + dpu_index=dpuhost.dpu_index, + ) + with open(ha_set_file) as f: ha_set_data = json.load(f)["DASH_HA_SET_CONFIG_TABLE"] @@ -136,8 +150,23 @@ def activate_dash_ha(duthost, dpuhost, localhost, ptfhost, setup_gnmi_server, logger.info(f"HA: Activate completed for {duthost.hostname}") +def verify_primary_standalone_traffic(ptfadapter, dash_pl_config): + primary_config = dash_pl_config[0] + send_pkt, exp_pkt = outbound_pl_packets(primary_config, "vxlan") + + logger.info("HA: verify PL traffic sent to primary NPU while primary is standalone") + ptfadapter.dataplane.flush() + testutils.send(ptfadapter, primary_config[LOCAL_PTF_INTF], send_pkt, count=1) + testutils.verify_packet_any_port( + ptfadapter, + exp_pkt, + primary_config[REMOTE_PTF_RECV_INTF], + ) + + def test_ha_launch_with_no_peer(request, duthosts, dpuhosts, localhost, ptfhost, setup_ha_config, - ha_owner, setup_gnmi_server, primary_vdpu_key, standby_vdpu_key): + ha_owner, setup_gnmi_server, primary_vdpu_key, standby_vdpu_key, + setup_dash_pl_pipeline, ptfadapter, dash_pl_config): logger.info("HA: activate only primary") try: @@ -148,6 +177,7 @@ def test_ha_launch_with_no_peer(request, duthosts, dpuhosts, localhost, ptfhost, pytest_assert(verify_ha_state(duthosts[0], scope_key=primary_vdpu_key, expected_state="standalone", timeout=150), "HA: Primary state is not standalone") + verify_primary_standalone_traffic(ptfadapter, dash_pl_config) logger.info("HA: activate standby with standalone primary") setup_dash_ha(duthosts[1], dpuhosts, localhost, ptfhost, setup_gnmi_server, ha_owner, role_index=1) diff --git a/tests/ha/test_ha_link_failure.py b/tests/ha/test_ha_link_failure.py index a688e3200a2..4483f07ca1e 100644 --- a/tests/ha/test_ha_link_failure.py +++ b/tests/ha/test_ha_link_failure.py @@ -1,6 +1,5 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time @@ -11,12 +10,17 @@ REMOTE_PTF_SEND_INTF, NPU_DATAPLANE_PORT ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets, inbound_pl_packets from tests.common.helpers.assertions import pytest_assert +from tests.ha.conftest import apply_dash_pl_pipeline_config from ha_dash_flow_utils import compare_flow_tables from ha_utils import set_dash_ha_scope, activate_secondary_dash_ha, verify_ha_state -from ha_link_utils import add_acl_link_drop, remove_acl_link_drop_table +from ha_link_utils import ( + add_acl_link_drop, + remove_acl_link_drop_table, + shutdown_dpu_dataplane_port, + startup_dpu_dataplane_port +) logger = logging.getLogger(__name__) @@ -54,46 +58,7 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Start DASH config on {duthost.hostname} dpu {dpuhost.dpu_index} with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield @@ -270,3 +235,164 @@ def link_ha_action(): else: pytest.fail(f"{link_fail} with {traffic} test error. Sent: {send_count}," f" not received: {failed_count} loss: {percentage_loss}, threshold: {threshold_loss}") + + +@pytest.mark.parametrize( + "standby_link_fail", [True, False], + ids=["Standby_Link_Fail", "Primary_Link_Fail"] +) +@pytest.mark.parametrize( + "traffic_to_standby", [True, False], + ids=["Standby_Traffic", "Primary_Traffic"] +) +def test_ha_link_down( + ptfadapter, + localhost, + duthosts, + dpuhosts, + ptfhost, + activate_dash_ha_from_json, + dash_pl_config, + standby_link_fail, + traffic_to_standby, + primary_vdpu_key, + standby_vdpu_key, + ha_owner +): + encap_proto = "vxlan" + initial_send_count = 100 + delay = 1.0 / RATE_PPS + rcv_outbound_pl_ports = dash_pl_config[0][REMOTE_PTF_RECV_INTF] + dash_pl_config[1][REMOTE_PTF_RECV_INTF] + + if traffic_to_standby: + vm_to_dpu_pkt, exp_dpu_to_pe_pkt = outbound_pl_packets(dash_pl_config[1], encap_proto) + pe_to_dpu_pkt, exp_dpu_to_vm_pkt = inbound_pl_packets(dash_pl_config[1]) + else: + vm_to_dpu_pkt, exp_dpu_to_pe_pkt = outbound_pl_packets(dash_pl_config[0], encap_proto) + pe_to_dpu_pkt, exp_dpu_to_vm_pkt = inbound_pl_packets(dash_pl_config[0]) + + _, exp_dpu_to_vm_pkt_standby = inbound_pl_packets(dash_pl_config[1]) + packet_sending_event = threading.Event() + stop_link_action_event = threading.Event() + + send_count = 0 + failed_count = 0 + + def link_ha_action(): + # wait for packets sending started, then shut down the DPU dataplane interface + while not packet_sending_event.is_set(): + if stop_link_action_event.wait(0.2): + return + if standby_link_fail: + logger.info(f"Shut down standby DPU dataplane interface, pkt sent {send_count}") + shutdown_dpu_dataplane_port(dpuhosts[1]) + else: + logger.info(f"Shut down primary DPU dataplane interface, pkt sent {send_count}") + shutdown_dpu_dataplane_port(dpuhosts[0]) + logger.info(f"After DPU dataplane interface down, pkt sent {send_count}") + + t = threading.Thread(target=link_ha_action, name="link_ha_action_thread") + t.start() + t_max = time.time() + 60 + reached_max_time = False + ptfadapter.dataplane.flush() + time.sleep(1) + rcv_inbound_pl_ports = [dash_pl_config[0][LOCAL_PTF_INTF], dash_pl_config[1][LOCAL_PTF_INTF]] + while not reached_max_time: + # After we send initial_send_count packets, awake link_ha_action thread + if send_count == initial_send_count: + logger.info("Awake link down HA action thread") + packet_sending_event.set() + + try: + if traffic_to_standby: + if send_count == 0: + logger.info("Send first outbound packet to standby") + testutils.send(ptfadapter, dash_pl_config[1][LOCAL_PTF_INTF], vm_to_dpu_pkt, 1) + testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) + if send_count == 0: + logger.info("First outbound packet received") + logger.info("Send first inbound packet to standby") + try: + testutils.send(ptfadapter, dash_pl_config[1][REMOTE_PTF_SEND_INTF], pe_to_dpu_pkt, 1) + if failed_count > 0: + testutils.verify_packet(ptfadapter, exp_dpu_to_vm_pkt_standby, + dash_pl_config[1][LOCAL_PTF_INTF]) + else: + testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_vm_pkt, rcv_inbound_pl_ports) + except Exception as e: + logger.info(f"inbound pkt dropped: {e}") + failed_count += 1 + if send_count == 0: + logger.info("First packets verified to standby - compare flows") + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) + pytest_assert(flow_op, "Expected identical flow tables on primary and standby") + + else: + if send_count == 0: + logger.info("Send first outbound packet to primary") + testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_to_dpu_pkt, 1) + testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) + if send_count == 0: + logger.info("First outbound packet received") + logger.info("Send first inbound packet to primary") + try: + testutils.send(ptfadapter, dash_pl_config[0][REMOTE_PTF_SEND_INTF], pe_to_dpu_pkt, 1) + if failed_count > 0: + testutils.verify_packet(ptfadapter, exp_dpu_to_vm_pkt_standby, + dash_pl_config[1][LOCAL_PTF_INTF]) + else: + testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_vm_pkt, rcv_inbound_pl_ports) + if send_count == 0: + logger.info("First inbound packet received") + except Exception as e: + logger.warning(f"inbound pkt dropped: {e}") + failed_count += 1 + except Exception as e: + logger.info(f"outbound pkt dropped after {send_count} pkts, exception {e}") + if send_count == 0: + logger.error(f"pkt dropped exception {e}") + pytest.fail("HA link down test error: no packets were received") + failed_count += 1 + + send_count += 1 + time.sleep(delay) + reached_max_time = time.time() > t_max + + stop_link_action_event.set() + t.join(timeout=5) + pytest_assert(not t.is_alive(), "link_ha_action thread did not exit in time") + + if send_count < initial_send_count: + pytest.fail( + f"HA link down test error: sent only {send_count} packets in test window, " + f"requires at least {initial_send_count} to trigger link down action" + ) + + time.sleep(2) + if standby_link_fail: + startup_dpu_dataplane_port(dpuhosts[1]) + else: + startup_dpu_dataplane_port(dpuhosts[0]) + # take system out of split-brain + pytest_assert(verify_ha_state(duthosts[0], primary_vdpu_key, "standalone"), + "Primary HA state is not standalone") + restore_ha_state(localhost, ptfhost, duthosts[1], standby_vdpu_key=standby_vdpu_key, ha_owner=ha_owner) + + traffic = "traffic to standby" if traffic_to_standby else "traffic to primary" + link_fail = "Standby link down" if standby_link_fail else "Primary link down" + if standby_link_fail: + if failed_count > 0: + pytest.fail(f"{link_fail} with {traffic} test error:" + f"{failed_count} packets not received {send_count} packets sent.") + else: + logger.info(f"{link_fail} with {traffic} test OK. All {send_count} packets sent were received.") + else: + threshold_loss = RATE_PPS * TRAFFIC_LOSS_DURATION + percentage_loss = (failed_count / send_count) * 100 + if (failed_count < threshold_loss): + logger.info(f"{link_fail} with {traffic} test OK. Sent: {send_count}," + f" not received: {failed_count}, loss: {percentage_loss}, threshold: {threshold_loss}") + else: + pytest.fail(f"{link_fail} with {traffic} test error. Sent: {send_count}," + f" not received: {failed_count} loss: {percentage_loss}, threshold: {threshold_loss}") diff --git a/tests/ha/test_ha_npu_process_crash.py b/tests/ha/test_ha_npu_process_crash.py index 34e408aca8d..1088c8b397c 100644 --- a/tests/ha/test_ha_npu_process_crash.py +++ b/tests/ha/test_ha_npu_process_crash.py @@ -226,8 +226,7 @@ def _run( ) # Wait for both DUTs to reach their initial expected HA states before - # the crash. Both DUTs show "active" in local_acked_asic_ha_state - # because the DPU HA dataplane operates in active/active mode. + # the crash. logger.info("Waiting for initial HA states to stabilize before crash") verify_ha_state_converged( crash_duthost, crash_scope_key, expected_ha_state_after_crash @@ -460,7 +459,8 @@ def test_crash_active_npu_traffic_on_active( localhost, ptfhost, activate_dash_ha_from_json, primary_vdpu_key, - standby_vdpu_key + standby_vdpu_key, + ha_owner ): self._run( process_name=process_name, container=container, @@ -469,7 +469,7 @@ def test_crash_active_npu_traffic_on_active( expected_ha_state_after_crash="active", verify_duthost=standby_dut, verify_scope_key=standby_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify="active" if ha_owner == "dpu" else "standby", ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=0, duthosts=duthosts, localhost=localhost, ptfhost=ptfhost, @@ -484,6 +484,7 @@ def test_crash_active_npu_traffic_on_standby( ptfadapter, dash_pl_config, localhost, ptfhost, activate_dash_ha_from_json, + ha_owner ): self._run( process_name=process_name, container=container, @@ -492,7 +493,7 @@ def test_crash_active_npu_traffic_on_standby( expected_ha_state_after_crash="active", verify_duthost=standby_dut, verify_scope_key=standby_vdpu_key, - expected_ha_state_verify="active", + expected_ha_state_verify="active" if ha_owner == "dpu" else "standby", ptfadapter=ptfadapter, dash_pl_config=dash_pl_config, traffic_dut_index=1, duthosts=duthosts, localhost=localhost, ptfhost=ptfhost, @@ -507,12 +508,13 @@ def test_crash_standby_npu_traffic_on_active( ptfadapter, dash_pl_config, localhost, ptfhost, activate_dash_ha_from_json, + ha_owner ): self._run( process_name=process_name, container=container, crash_duthost=standby_dut, crash_scope_key=standby_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash="active" if ha_owner == "dpu" else "standby", verify_duthost=primary_dut, verify_scope_key=primary_vdpu_key, expected_ha_state_verify="active", @@ -530,12 +532,13 @@ def test_crash_standby_npu_traffic_on_standby( ptfadapter, dash_pl_config, localhost, ptfhost, activate_dash_ha_from_json, + ha_owner ): self._run( process_name=process_name, container=container, crash_duthost=standby_dut, crash_scope_key=standby_vdpu_key, - expected_ha_state_after_crash="active", + expected_ha_state_after_crash="active" if ha_owner == "dpu" else "standby", verify_duthost=primary_dut, verify_scope_key=primary_vdpu_key, expected_ha_state_verify="active", diff --git a/tests/ha/test_ha_npu_reboot.py b/tests/ha/test_ha_npu_reboot.py index 01fceac18c2..2adb65c53b9 100644 --- a/tests/ha/test_ha_npu_reboot.py +++ b/tests/ha/test_ha_npu_reboot.py @@ -1,6 +1,5 @@ import logging from multiprocessing.pool import ThreadPool -import concurrent.futures import configs.privatelink_config as pl import ptf.testutils as testutils @@ -14,20 +13,20 @@ REMOTE_PTF_RECV_INTF ) from gnmi_utils import ( - apply_messages, apply_gnmi_cert, generate_gnmi_cert, recover_gnmi_cert ) from packets import outbound_pl_packets from tests.common.utilities import wait_until -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from tests.common.helpers.assertions import pytest_assert, pytest_require as pt_require from tests.common.platform.processes_utils import wait_critical_processes -from ha_dash_flow_utils import compare_flow_tables_pdsctl +from ha_dash_flow_utils import compare_flow_tables from tests.common.reboot import reboot_smartswitch, wait_for_startup from tests.ha.conftest import get_interface_ip from tests.ha.ha_dpu_utils import CHECK_DPU_STATE_TIMEOUT, CHECK_DPU_STATE_TIME_INT, check_dpu_up_state +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -43,11 +42,6 @@ INITIAL_SEND_COUNT = 100 -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - @pytest.fixture(scope="function") def setup_gnmi_server(duthosts, localhost, ptfhost, skip_cert_cleanup): for duthost in duthosts: @@ -118,58 +112,16 @@ def common_setup_teardown( setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server, - set_vxlan_udp_sport_range, + ensure_vxlan_udp_sport_range, setup_npu_dpu # noqa: F811 ): if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Starting DASH configuration on {duthost.hostname}" - "dpu {dpuhost.dpu_index} with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - # Map the reload_config_for_host function to the dpuhosts list - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) """ @@ -255,7 +207,7 @@ def npu_ha_action(): testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) if send_count == 0: logger.info("First packet to standby received - compare flows") - flow_op = compare_flow_tables_pdsctl(dpuhosts[0], dpuhosts[1]) + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) pytest_assert(flow_op, "Expected identical flow tables on primary and standby") else: @@ -265,7 +217,7 @@ def npu_ha_action(): testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) if send_count == 0: logger.info("First packet to primary received - compare flows") - flow_op = compare_flow_tables_pdsctl(dpuhosts[0], dpuhosts[1]) + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) pytest_assert(flow_op, "Expected identical flow tables on primary and standby") except Exception as e: if failed_count == 0: diff --git a/tests/ha/test_ha_planned_shutdown.py b/tests/ha/test_ha_planned_shutdown.py index d099aaeb86b..8d82da5de2a 100644 --- a/tests/ha/test_ha_planned_shutdown.py +++ b/tests/ha/test_ha_planned_shutdown.py @@ -1,6 +1,5 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest import time @@ -8,9 +7,9 @@ import queue from tests.common.helpers.assertions import pytest_assert from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF -from gnmi_utils import apply_messages from packets import outbound_pl_packets from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from ha_dash_flow_utils import compare_flow_tables, compare_flow_tables_pdsctl from ha_utils import activate_primary_dash_ha, activate_secondary_dash_ha, \ verify_ha_state, set_dash_ha_scope @@ -42,50 +41,12 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"configure on {duthost.hostname} dpu {dpuhost.dpu_index} {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - config_reload(dpuhost, safe_reload=True, yang_validate=False) + for dpuhost in dpuhosts: + config_reload(dpuhost, safe_reload=True, yang_validate=False) def test_ha_planned_shutdown( @@ -211,6 +172,8 @@ def standby_ha_action(): testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_post_sd, 1) testutils.verify_packet_any_port(ptfadapter, exp_post_sd, rcv_outbound_pl_ports) + set_dash_ha_scope(localhost, duthosts[1], ptfhost, standby_vdpu_key, "dead", ha_owner, disabled=True) + # Re-activate standby pytest_assert(activate_secondary_dash_ha(localhost, duthosts[1], ptfhost, standby_vdpu_key, "activate_role", owner=ha_owner), "Failed to re-activate HA on standby") diff --git a/tests/ha/test_ha_planned_shutdown_fnic.py b/tests/ha/test_ha_planned_shutdown_fnic.py index e74b5752dfa..447512f4187 100644 --- a/tests/ha/test_ha_planned_shutdown_fnic.py +++ b/tests/ha/test_ha_planned_shutdown_fnic.py @@ -1,5 +1,4 @@ import logging -import concurrent.futures import random import configs.privatelink_config as pl @@ -10,25 +9,31 @@ import queue from tests.common.helpers.assertions import pytest_assert from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF -from gnmi_utils import apply_messages from packets import outbound_pl_packets -from tests.common.config_reload import config_reload -from ha_utils import activate_primary_dash_ha, activate_secondary_dash_ha, \ - verify_ha_state, set_dash_ha_scope, set_dead_dash_ha_scope +from tests.ha.conftest import apply_dash_pl_pipeline_config +from ha_dash_flow_utils import compare_flow_tables, compare_flow_tables_pdsctl +from ha_utils import ( + activate_primary_dash_ha, + activate_secondary_dash_ha, + verify_ha_state, + set_dash_ha_scope, + set_dead_dash_ha_scope, + parallel_config_reload_dpuhosts, +) logger = logging.getLogger(__name__) +# Distinct inner UDP ports used only after standby shutdown to create a new +# flow on the standalone primary and verify it is bulk-synced to the standby. +POST_SHUTDOWN_INNER_SPORT = 50001 +POST_SHUTDOWN_INNER_DPORT = 50002 + pytestmark = [ pytest.mark.topology('t1-smartswitch-ha'), pytest.mark.skip_check_dut_health ] -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - @pytest.fixture(autouse=True, scope="function") def common_setup_teardown( localhost, @@ -46,58 +51,11 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_FNIC_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - **pl.TUNNEL1_CONFIG, - } - logger.info(f"configure on {duthost.hostname} dpu {dpuhost.dpu_index} {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_VNET_MAPPING_CONFIG, - **pl.VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT, - } - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - if 'pensando' not in dpuhost.facts['asic_type']: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - **pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_FNIC_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost, floating_nic=True) yield - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) def test_ha_planned_shutdown( @@ -119,57 +77,61 @@ def test_ha_planned_shutdown( rcv_outbound_pl_ports = dash_pl_config[0][REMOTE_PTF_RECV_INTF] + dash_pl_config[1][REMOTE_PTF_RECV_INTF] - packet_sending_flag = queue.Queue(1) - - def primary_ha_action(): - # wait for packets sending started, then set primary to dead - while packet_sending_flag.empty() or (not packet_sending_flag.get()): - time.sleep(0.2) - logging.info("HA: Set primary to dead") - set_dead_dash_ha_scope(localhost, duthosts[0], ptfhost, primary_vdpu_key) - - t = threading.Thread(target=primary_ha_action, name="primary_ha_action_thread") - t.start() - t_max = time.time() + 60 - # Calculate the delay between packets based on the desired rate - reached_max_time = False - ptfadapter.dataplane.flush() - time.sleep(1) - send_count = 0 - while not reached_max_time: - sport = random.randint(49152, 65535) - dport = random.randint(49152, 65535) - vm_to_dpu_pkt, exp_dpu_to_pe_pkt = outbound_pl_packets( - dash_pl_config[0], encap_proto, floating_nic=True, - inner_sport=sport, inner_dport=dport, vni=pl.ENI_TRUSTED_VNI - ) - testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_to_dpu_pkt, 1) - testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) - if send_count == 0: - logger.info("HA: First packet received") - send_count += 1 - # After we send initial_send_count packets, awake perform_ha_action thread - if send_count == initial_send_count: - logging.info("HA: awake action thread") - packet_sending_flag.put(True) - - time.sleep(delay) - reached_max_time = time.time() > t_max - - t.join() - time.sleep(2) - - pytest_assert(verify_ha_state(duthosts[0], primary_vdpu_key, "dead"), - "Primary HA state is not dead") - pytest_assert(verify_ha_state(duthosts[1], standby_vdpu_key, "standalone"), - "Standby HA state is not standalone") - - logging.info(f"HA: Primary shutdown all {send_count} packets received") - - # Re-activate primary - set_dash_ha_scope(localhost, duthosts[0], ptfhost, primary_vdpu_key, "dead", ha_owner, disabled=True) - pytest_assert(activate_primary_dash_ha(localhost, duthosts[0], ptfhost, primary_vdpu_key, "activate_role"), - "Failed to re-activate HA on primary") + if ha_owner == "dpu": + # shutdown active HA Scope is only applicable to DPU-driven HA + packet_sending_flag = queue.Queue(1) + + def primary_ha_action(): + # wait for packets sending started, then set primary to dead + while packet_sending_flag.empty() or (not packet_sending_flag.get()): + time.sleep(0.2) + logging.info("HA: Set primary to dead") + set_dead_dash_ha_scope(localhost, duthosts[0], ptfhost, primary_vdpu_key, ha_owner) + + t = threading.Thread(target=primary_ha_action, name="primary_ha_action_thread") + t.start() + t_max = time.time() + 60 + # Calculate the delay between packets based on the desired rate + reached_max_time = False + ptfadapter.dataplane.flush() + time.sleep(1) + send_count = 0 + while not reached_max_time: + sport = random.randint(49152, 65535) + dport = random.randint(49152, 65535) + vm_to_dpu_pkt, exp_dpu_to_pe_pkt = outbound_pl_packets( + dash_pl_config[0], encap_proto, floating_nic=True, + inner_sport=sport, inner_dport=dport, vni=pl.ENI_TRUSTED_VNI + ) + testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_to_dpu_pkt, 1) + testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) + if send_count == 0: + logger.info("HA: First packet received - compare flows") + flow_op = compare_flow_tables_pdsctl(dpuhosts[0], dpuhosts[1]) + pytest_assert(flow_op, "Expected identical flow tables on primary and standby") + send_count += 1 + # After we send initial_send_count packets, awake perform_ha_action thread + if send_count == initial_send_count: + logging.info("HA: awake action thread") + packet_sending_flag.put(True) + + time.sleep(delay) + reached_max_time = time.time() > t_max + + t.join() + time.sleep(2) + + pytest_assert(verify_ha_state(duthosts[0], primary_vdpu_key, "dead"), + "Primary HA state is not dead") + pytest_assert(verify_ha_state(duthosts[1], standby_vdpu_key, "standalone"), + "Standby HA state is not standalone") + + logging.info(f"HA: Primary shutdown all {send_count} packets received") + + # Re-activate primary + set_dash_ha_scope(localhost, duthosts[0], ptfhost, primary_vdpu_key, "dead", ha_owner, disabled=True) + pytest_assert(activate_primary_dash_ha(localhost, duthosts[0], ptfhost, primary_vdpu_key, "activate_role"), + "Failed to re-activate HA on primary") packet_sending_flag = queue.Queue(1) @@ -178,7 +140,7 @@ def standby_ha_action(): while packet_sending_flag.empty() or (not packet_sending_flag.get()): time.sleep(0.2) logging.info("HA: Set standby to dead") - set_dead_dash_ha_scope(localhost, duthosts[1], ptfhost, standby_vdpu_key) + set_dead_dash_ha_scope(localhost, duthosts[1], ptfhost, standby_vdpu_key, ha_owner) t = threading.Thread(target=standby_ha_action, name="standby_ha_action_thread") t.start() @@ -198,7 +160,9 @@ def standby_ha_action(): testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_to_dpu_pkt, 1) testutils.verify_packet_any_port(ptfadapter, exp_dpu_to_pe_pkt, rcv_outbound_pl_ports) if send_count == 0: - logger.info("HA: First packet received") + logger.info("HA: First packet received - compare flows") + flow_op = compare_flow_tables(dpuhosts[0], dpuhosts[1]) + pytest_assert(flow_op, "Expected identical flow tables on primary and standby") send_count += 1 # After we send initial_send_count packets, awake perform_ha_action thread if send_count == initial_send_count: @@ -217,7 +181,26 @@ def standby_ha_action(): logging.info(f"HA: standby shutdown all {send_count} packets received") + logger.info( + "HA: Post-shutdown - send outbound packet with inner_sport=%s then compare flow tables", + POST_SHUTDOWN_INNER_SPORT, + ) + ptfadapter.dataplane.flush() + time.sleep(1) + vm_post_sd, exp_post_sd = outbound_pl_packets( + dash_pl_config[0], encap_proto, floating_nic=True, + inner_sport=POST_SHUTDOWN_INNER_SPORT, inner_dport=POST_SHUTDOWN_INNER_DPORT, + vni=pl.ENI_TRUSTED_VNI + ) + testutils.send(ptfadapter, dash_pl_config[0][LOCAL_PTF_INTF], vm_post_sd, 1) + testutils.verify_packet_any_port(ptfadapter, exp_post_sd, rcv_outbound_pl_ports) + # Re-activate standby set_dash_ha_scope(localhost, duthosts[1], ptfhost, standby_vdpu_key, "dead", ha_owner, disabled=True) pytest_assert(activate_secondary_dash_ha(localhost, duthosts[1], ptfhost, standby_vdpu_key, "activate_role", owner=ha_owner), "Failed to re-activate HA on standby") + + flow_post = compare_flow_tables( + dpuhosts[0], dpuhosts[1], verbose=True, flow_state=True + ) + pytest_assert(flow_post, "Expected identical flow tables after launch from standalone (bulk sync)") diff --git a/tests/ha/test_ha_planned_swo.py b/tests/ha/test_ha_planned_swo.py index 77794143feb..1e87816328d 100644 --- a/tests/ha/test_ha_planned_swo.py +++ b/tests/ha/test_ha_planned_swo.py @@ -1,25 +1,22 @@ -import concurrent.futures import logging import queue import random import threading import time -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from constants import ( LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF, VXLAN_UDP_BASE_SRC_PORT, VXLAN_UDP_SRC_PORT_MASK, ) -from gnmi_utils import apply_messages from packets import outbound_pl_packets from ha_dash_flow_utils import compare_flow_tables -from ha_utils import verify_ha_state, set_dash_ha_scope +from ha_utils import verify_ha_state, set_dash_ha_scope, parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -51,55 +48,10 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"configure on {duthost.hostname} dpu {dpuhost.dpu_index} {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - - def _reload(host): - logger.info(f"config reload on {host.hostname}") - config_reload(host, safe_reload=True, yang_validate=False) - - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - list(executor.map(_reload, dpuhosts)) + parallel_config_reload_dpuhosts(dpuhosts) def _planned_swo_phase( diff --git a/tests/ha/test_ha_repairing_dpu.py b/tests/ha/test_ha_repairing_dpu.py new file mode 100644 index 00000000000..61da8c6342c --- /dev/null +++ b/tests/ha/test_ha_repairing_dpu.py @@ -0,0 +1,785 @@ +import json +import logging +import os +import threading +import time + +import configs.privatelink_config as pl +import ptf.testutils as testutils +import pytest +from constants import ( + LOCAL_PTF_INTF, + REMOTE_PTF_RECV_INTF, + VXLAN_UDP_BASE_SRC_PORT, + VXLAN_UDP_SRC_PORT_MASK, +) +from packets import outbound_pl_packets +from tests.common.devices.duthosts import DutHosts +from tests.common.dash_utils import apply_swssconfig_file +from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.utilities import InterruptableThread +from tests.conftest import get_specified_dpus, get_target_hostname, is_parallel_leader +from tests.ha.conftest import apply_dash_pl_pipeline_config +from ha_gnmi import apply_ha_messages, ha_scope_config, ha_set_config +from ha_utils import ( + parallel_config_reload_dpuhosts, + program_eni_pl_on_dpu, + set_dash_ha_scope, + verify_ha_state, + wait_for_pending_operation_id, +) + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('t1-smartswitch-ha'), + pytest.mark.skip_check_dut_health, +] + +TRAFFIC_SEND_INTERVAL = 0.1 +MAX_TRAFFIC_LOSS_PCT = 5.0 +PL_VERIFY_TIMEOUT = 10 + + +def _requested_dpuhosts(host_nodes, request): + requested_dpuhosts = get_specified_dpus(request) + nodes_by_hostname = {node.hostname: node for node in host_nodes} + missing_dpuhosts = [ + hostname for hostname in requested_dpuhosts if hostname not in nodes_by_hostname + ] + pytest_require( + not missing_dpuhosts, + f"Requested DPU hosts were not initialized: {missing_dpuhosts}", + ) + return [nodes_by_hostname[hostname] for hostname in requested_dpuhosts] + + +def _dpuhost_matches_duthost(dpuhost, duthost): + return dpuhost.hostname == duthost.hostname or dpuhost.hostname.startswith( + f"{duthost.hostname}-" + ) + + +def _repair_index(repair_target): + return 0 if repair_target == "primary" else 1 + + +def _peer_index(repair_target): + return 1 - _repair_index(repair_target) + + +def _repair_title(repair_target): + return repair_target.capitalize() + + +def _peer_title(repair_target): + return "Standby" if repair_target == "primary" else "Primary" + + +def _replacement_desired_ha_state(repair_target): + return "active" if repair_target == "primary" else "unspecified" + + +def _replacement_expected_state(repair_target, ha_owner): + if repair_target == "primary": + return "active" + return "active" if ha_owner == "dpu" else "standby" + + +def _expected_peer_state_after_repair(repair_target, ha_owner): + if repair_target == "primary": + return "active" if ha_owner == "dpu" else "standby" + return "active" + + +def _select_replacement_dpuhost(requested_dpuhosts, duthost_to_replace): + pytest_require( + len(requested_dpuhosts) in (3, 4), + "HA repairing tests require exactly 3 or 4 requested DPU hosts", + ) + + if len(requested_dpuhosts) == 3: + return requested_dpuhosts[2] + + replacement_candidates = requested_dpuhosts[-2:] + for dpuhost in replacement_candidates: + if _dpuhost_matches_duthost(dpuhost, duthost_to_replace): + return dpuhost + + pytest_require( + False, + "The last two requested DPU hosts {} do not include a replacement on {}".format( + [dpuhost.hostname for dpuhost in replacement_candidates], + duthost_to_replace.hostname, + ), + ) + + +@pytest.fixture(scope="session") +def dpuhosts( + enhance_inventory, + ansible_adhoc, + tbinfo, + request, + enable_nat_for_dpuhosts, + duthosts, +): + """Return all requested DPU hosts in CLI order for the repair test module.""" + del enhance_inventory, enable_nat_for_dpuhosts + + host = DutHosts( + ansible_adhoc, + tbinfo, + request, + get_specified_dpus(request), + target_hostname=get_target_hostname(request), + is_parallel_leader=is_parallel_leader(request), + ) + requested_dpuhosts = _requested_dpuhosts(host.nodes, request) + pytest_require( + len(requested_dpuhosts) in (3, 4), + "HA repairing tests require exactly 3 or 4 requested DPU hosts", + ) + pytest_require( + _dpuhost_matches_duthost(requested_dpuhosts[0], duthosts[0]), + "The first requested DPU host must belong to {}".format(duthosts[0].hostname), + ) + pytest_require( + _dpuhost_matches_duthost(requested_dpuhosts[1], duthosts[1]), + "The second requested DPU host must belong to {}".format(duthosts[1].hostname), + ) + return requested_dpuhosts + + +@pytest.fixture(autouse=True, scope="module") +def require_replacement_dpu(dpuhosts): + pytest_require( + len(dpuhosts) in (3, 4), + "HA repairing tests require the HA pair plus one or two replacement DPU candidates", + ) + + +@pytest.fixture(params=["primary", "standby"], ids=["primary", "standby"]) +def repair_target(request, ha_owner): + if request.param == "primary" and ha_owner != "dpu": + pytest.skip("Re-pairing the Active (primary) DPU is only supported for DPU-driven mode") + return request.param + + +@pytest.fixture(scope="function") +def selected_dpuhosts(dpuhosts, duthosts, repair_target): + replacement_dpuhost = _select_replacement_dpuhost( + dpuhosts, + duthosts[_repair_index(repair_target)], + ) + return [dpuhosts[0], dpuhosts[1], replacement_dpuhost] + + +def _all_recv_ports(dash_pl_config): + """Combine REMOTE_PTF_RECV_INTF from both DUTs so packets exiting either switch are counted.""" + ports = list(dash_pl_config[0][REMOTE_PTF_RECV_INTF]) + for port in dash_pl_config[1][REMOTE_PTF_RECV_INTF]: + if port not in ports: + ports.append(port) + return ports + + +def _send_continuous_pl_traffic(ptfadapter, send_config, recv_ports, stop_event, results): + sent = 0 + received = 0 + send_pkt, exp_pkt = outbound_pl_packets(send_config, "vxlan") + while not stop_event.is_set(): + try: + testutils.send(ptfadapter, send_config[LOCAL_PTF_INTF], send_pkt, count=1) + sent += 1 + try: + testutils.verify_packet_any_port( + ptfadapter, + exp_pkt, + recv_ports, + timeout=1, + ) + received += 1 + except AssertionError: + logger.debug("Packet not received") + except Exception as error: + logger.debug(f"Traffic sender: {error}") + time.sleep(TRAFFIC_SEND_INTERVAL) + results["sent"] = sent + results["received"] = received + + +def _verify_baseline_pl_traffic(ptfadapter, send_config, recv_ports): + send_pkt, exp_pkt = outbound_pl_packets(send_config, "vxlan") + ptfadapter.dataplane.flush() + testutils.send(ptfadapter, send_config[LOCAL_PTF_INTF], send_pkt, count=1) + testutils.verify_packet_any_port( + ptfadapter, + exp_pkt, + recv_ports, + timeout=PL_VERIFY_TIMEOUT, + ) + + +def _replacement_context(duthosts, selected_dpuhosts): + replacement_dpuhost = selected_dpuhosts[2] + + for dut_index, duthost in enumerate(duthosts): + if _dpuhost_matches_duthost(replacement_dpuhost, duthost): + return duthost, dut_index, replacement_dpuhost + + raise ValueError( + "Unable to map replacement DPU '{}' to a DUT in {}".format( + replacement_dpuhost.hostname, + [duthost.hostname for duthost in duthosts], + ) + ) + + +def _replacement_vdpu_id(duthosts, selected_dpuhosts): + _, replacement_dut_index, replacement_dpuhost = _replacement_context( + duthosts, + selected_dpuhosts, + ) + return f"vdpu{replacement_dut_index}_{replacement_dpuhost.dpu_index}" + + +def _apply_vxlan_udp_sport_range(dpuhosts): + vxlan_sport_config = [ + { + "SWITCH_TABLE:switch": { + "vxlan_sport": VXLAN_UDP_BASE_SRC_PORT, + "vxlan_mask": VXLAN_UDP_SRC_PORT_MASK, + }, + "OP": "SET", + } + ] + + logger.info(f"Setting VXLAN source port config: {vxlan_sport_config}") + config_path = "/tmp/vxlan_sport_config.json" + for dpuhost in dpuhosts: + dpuhost.copy(content=json.dumps(vxlan_sport_config, indent=4), dest=config_path, verbose=False) + apply_swssconfig_file(dpuhost, config_path) + if 'pensando' in dpuhost.facts['asic_type']: + logger.warning("Applying Pensando DPU VXLAN sport workaround") + dpuhost.shell("pdsctl debug update device --vxlan-port 4789 --vxlan-src-ports 5120-5247") + + +@pytest.fixture(scope="function") +def repair_runtime_state(): + return {"replacement_scope_programmed": False} + + +def _cleanup_programmed_dpu(localhost, ptfhost, duthost, dpuhost): + logger.info( + f"Removing DPU PL programming on {dpuhost.hostname}" + ) + apply_dash_pl_pipeline_config( + localhost, + [duthost], + [dpuhost], + ptfhost, + set_db=False, + wait_after_apply=1, + ) + + +@pytest.fixture(scope="function") +def set_vxlan_udp_sport_range(selected_dpuhosts, skip_config): + if skip_config: + yield + return + + _apply_vxlan_udp_sport_range(selected_dpuhosts) + yield + + +@pytest.fixture(scope="function") +def setup_npu_dpu(add_npu_static_routes, duthosts, selected_dpuhosts, skip_config): + del add_npu_static_routes + if skip_config: + yield + return + + for dut_index, duthost in enumerate(duthosts): + dpuhost = selected_dpuhosts[dut_index] + dpuhost.shell(f'ip route replace {duthost.mgmt_ip}/32 via 169.254.200.254') + interfaces = dpuhost.shell("show ip int")["stdout"] + dpu_commands = [] + if "Loopback0" not in interfaces: + dpu_commands.append("config loopback add Loopback0") + dpu_commands.append(f"config int ip add Loopback0 {pl.APPLIANCE_VIP}/32") + dpuhost.shell_cmds(cmds=dpu_commands) + + yield + + +def _activate_replacement_dpu( + localhost, + replacement_duthost, + ptfhost, + scope_key, + owner, + repair_target, +): + fields = { + "version": "1", + "disabled": False, + "desired_ha_state": _replacement_desired_ha_state(repair_target), + "owner": owner, + } + vdpu_id_part, ha_set_id_part = scope_key.split(":", 1) + activation_messages = ha_scope_config( + vdpu_id=vdpu_id_part, + ha_set_id=ha_set_id_part, + **fields, + ) + apply_ha_messages( + localhost=localhost, + duthost=replacement_duthost, + ptfhost=ptfhost, + messages=activation_messages, + ) + + pending_id = wait_for_pending_operation_id( + replacement_duthost, + scope_key, + "activate_role", + timeout=60, + ) + if pending_id: + logger.info( + f"Replacement {repair_target} DPU {scope_key} pending id {pending_id}; approving activation" + ) + approval_messages = ha_scope_config( + vdpu_id=vdpu_id_part, + ha_set_id=ha_set_id_part, + approved_pending_operation_ids=[pending_id], + **fields, + ) + apply_ha_messages( + localhost=localhost, + duthost=replacement_duthost, + ptfhost=ptfhost, + messages=approval_messages, + ) + else: + logger.info( + f"Replacement {repair_target} DPU {scope_key} has no activate_role pending id; " + "continuing with direct state verification" + ) + + return verify_ha_state( + replacement_duthost, + scope_key, + _replacement_expected_state(repair_target, owner), + timeout=120, + interval=5, + ) + + +@pytest.fixture(autouse=True, scope="function") +def common_setup_teardown( + localhost, + duthosts, + ptfhost, + skip_config, + repair_runtime_state, + selected_dpuhosts, + setup_ha_config, + setup_dash_ha_from_json_func_scope, + ha_owner, + setup_gnmi_server, + set_vxlan_udp_sport_range, + setup_npu_dpu, +): + """ + Apply base DASH pipeline config on duthosts[0]/selected_dpuhosts[0] (primary) + and duthosts[1]/selected_dpuhosts[1] (standby). selected_dpuhosts[2] is the + scenario-selected replacement DPU and is left unconfigured until the test body + needs it. + """ + del setup_ha_config, setup_dash_ha_from_json_func_scope, setup_gnmi_server + del set_vxlan_udp_sport_range, setup_npu_dpu + + if skip_config: + return + + apply_dash_pl_pipeline_config( + localhost, + [duthosts[0], duthosts[1]], + [selected_dpuhosts[0], selected_dpuhosts[1]], + ptfhost, + ) + + yield + + replacement_duthost, _, replacement_dpuhost = _replacement_context(duthosts, selected_dpuhosts) + ha_set_id = "haset0_0" + replacement_scope_key = f"{_replacement_vdpu_id(duthosts, selected_dpuhosts)}:{ha_set_id}" + + try: + cleanup_targets = [ + (duthosts[0], selected_dpuhosts[0]), + (duthosts[1], selected_dpuhosts[1]), + (replacement_duthost, replacement_dpuhost), + ] + for duthost, dpuhost in cleanup_targets: + _cleanup_programmed_dpu( + localhost, + ptfhost, + duthost, + dpuhost, + ) + if repair_runtime_state["replacement_scope_programmed"]: + logger.info( + f"Setting replacement HA scope '{replacement_scope_key}' to dead on {replacement_duthost.hostname}" + ) + set_dash_ha_scope( + localhost, + replacement_duthost, + ptfhost, + replacement_scope_key, + "dead", + ha_owner, + disabled=True, + ) + + logger.info("Removing all DASH_HA_SCOPE_CONFIG_TABLE entries on both switches") + scope_targets = [ + (duthosts[0], f"vdpu0_{selected_dpuhosts[0].dpu_index}"), + (duthosts[1], f"vdpu1_{selected_dpuhosts[1].dpu_index}"), + ] + if repair_runtime_state["replacement_scope_programmed"]: + replacement_vdpu_id, _ = replacement_scope_key.split(":", 1) + scope_targets.append((replacement_duthost, replacement_vdpu_id)) + for duthost, vdpu_id in scope_targets: + scope_messages = ha_scope_config( + vdpu_id=vdpu_id, + ha_set_id=ha_set_id, + version="1", + disabled=True, + desired_ha_state="unspecified", + owner=ha_owner, + ) + apply_ha_messages( + localhost=localhost, + duthost=duthost, + ptfhost=ptfhost, + messages=scope_messages, + set_db=False, + ) + + logger.info("Removing all DASH_HA_SET_CONFIG_TABLE entries on both switches") + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.join(current_dir, "..", "common", "ha") + ha_set_file = os.path.join(base_dir, "dash_ha_set_config_table.json") + with open(ha_set_file) as ha_set_handle: + ha_set_data = json.load(ha_set_handle)["DASH_HA_SET_CONFIG_TABLE"] + for duthost in duthosts: + for key, fields in ha_set_data.items(): + ha_set_messages = ha_set_config(ha_set_id=key, **fields) + apply_ha_messages( + localhost=localhost, + duthost=duthost, + ptfhost=ptfhost, + messages=ha_set_messages, + set_db=False, + ) + finally: + parallel_config_reload_dpuhosts(selected_dpuhosts) + + +def _update_ha_set_with_replacement_dpu( + localhost, + duthosts, + ptfhost, + ha_owner, + old_vdpu_id, + new_vdpu_id, + old_duthost, + peer_duthost, + replacement_duthost, + repair_target, + ha_set_id="haset0_0", +): + """Replace the old DPU's HA programming with the replacement DPU. + + The peer switch (hosting the surviving DPU) keeps its existing scope and + only receives an updated DASH_HA_SET_CONFIG_TABLE. The old DPU's switch + has its old scope and old set deleted first, then the new set and new + scope (for the replacement DPU) are written. + """ + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.join(current_dir, "..", "common", "ha") + ha_set_file = os.path.join(base_dir, "dash_ha_set_config_table.json") + + with open(ha_set_file) as ha_set_handle: + ha_set_data = json.load(ha_set_handle)["DASH_HA_SET_CONFIG_TABLE"] + + ha_set_entry = ha_set_data.get(ha_set_id, {}) + vdpu_ids = ha_set_entry.get("vdpu_ids", []) + updated_vdpu_ids = [new_vdpu_id if vdpu_id == old_vdpu_id else vdpu_id for vdpu_id in vdpu_ids] + ha_set_entry["vdpu_ids"] = updated_vdpu_ids + + if repair_target == "primary": + if ha_set_entry.get("preferred_vdpu_id") == old_vdpu_id: + ha_set_entry["preferred_vdpu_id"] = new_vdpu_id + elif ha_set_entry.get("preferred_vdpu_id") == old_vdpu_id: + surviving_vdpu_ids = [vdpu_id for vdpu_id in updated_vdpu_ids if vdpu_id != new_vdpu_id] + if surviving_vdpu_ids: + ha_set_entry["preferred_vdpu_id"] = surviving_vdpu_ids[0] + + ha_set_data[ha_set_id] = ha_set_entry + + old_scope_key = f"{old_vdpu_id}:{ha_set_id}" + logger.info( + f"Removing old DASH_HA_SCOPE_CONFIG_TABLE '{old_scope_key}' on " + f"{old_duthost.hostname}" + ) + old_scope_messages = ha_scope_config( + vdpu_id=old_vdpu_id, + ha_set_id=ha_set_id, + version="1", + disabled=True, + desired_ha_state="unspecified", + owner=ha_owner, + ) + apply_ha_messages( + localhost=localhost, + duthost=old_duthost, + ptfhost=ptfhost, + messages=old_scope_messages, + set_db=False, + ) + + logger.info( + f"Removing old DASH_HA_SET_CONFIG_TABLE entries on {old_duthost.hostname}" + ) + for key, fields in ha_set_data.items(): + ha_set_messages = ha_set_config(ha_set_id=key, **fields) + apply_ha_messages( + localhost=localhost, + duthost=old_duthost, + ptfhost=ptfhost, + messages=ha_set_messages, + set_db=False, + ) + + logger.info( + f"Updating DASH_HA_SET_CONFIG_TABLE on peer {peer_duthost.hostname} " + f"with vdpu_ids={updated_vdpu_ids}" + ) + for key, fields in ha_set_data.items(): + ha_set_messages = ha_set_config(ha_set_id=key, **fields) + apply_ha_messages( + localhost=localhost, + duthost=peer_duthost, + ptfhost=ptfhost, + messages=ha_set_messages, + ) + + if replacement_duthost is not peer_duthost: + logger.info( + f"Creating DASH_HA_SET_CONFIG_TABLE on replacement {replacement_duthost.hostname} " + f"with vdpu_ids={updated_vdpu_ids}" + ) + for key, fields in ha_set_data.items(): + ha_set_messages = ha_set_config(ha_set_id=key, **fields) + apply_ha_messages( + localhost=localhost, + duthost=replacement_duthost, + ptfhost=ptfhost, + messages=ha_set_messages, + ) + + new_scope_key = f"{new_vdpu_id}:{ha_set_id}" + new_scope_fields = { + "version": "1", + "disabled": True, + "desired_ha_state": "unspecified", + "owner": ha_owner, + } + logger.info( + f"Programming new DPU scope '{new_scope_key}' with disabled state on {replacement_duthost.hostname}" + ) + vdpu_id_part, ha_set_id_part = new_scope_key.split(":", 1) + new_scope_messages = ha_scope_config( + vdpu_id=vdpu_id_part, + ha_set_id=ha_set_id_part, + **new_scope_fields, + ) + apply_ha_messages( + localhost=localhost, + duthost=replacement_duthost, + ptfhost=ptfhost, + messages=new_scope_messages, + ) + + +def test_ha_repairing_dpu( + localhost, + duthosts, + dpuhosts, + repair_runtime_state, + selected_dpuhosts, + ptfhost, + activate_dash_ha_from_json, + ha_owner, + ptfadapter, + dash_pl_config, + repair_target, +): + """ + Test replacement of either the primary or the standby DPU in a live HA set. + + Each parametrized case starts from a fresh HA setup and re-applies the DPU-side + configuration that is cleared by the per-case config_reload cleanup. + """ + del activate_dash_ha_from_json, dpuhosts + + ha_set_id = "haset0_0" + repair_index = _repair_index(repair_target) + peer_index = _peer_index(repair_target) + repair_title = _repair_title(repair_target) + peer_title = _peer_title(repair_target) + + replacement_duthost, _, replacement_dpuhost = _replacement_context( + duthosts, + selected_dpuhosts, + ) + + repair_vdpu_key = ( + f"vdpu{repair_index}_{selected_dpuhosts[repair_index].dpu_index}:{ha_set_id}" + ) + peer_vdpu_key = f"vdpu{peer_index}_{selected_dpuhosts[peer_index].dpu_index}:{ha_set_id}" + new_vdpu_id = _replacement_vdpu_id(duthosts, selected_dpuhosts) + new_vdpu_key = f"{new_vdpu_id}:{ha_set_id}" + expected_replacement_state = _replacement_expected_state(repair_target, ha_owner) + expected_peer_state = _expected_peer_state_after_repair(repair_target, ha_owner) + + pl_config = dash_pl_config[0] + recv_ports = _all_recv_ports(dash_pl_config) + _verify_baseline_pl_traffic(ptfadapter, pl_config, recv_ports) + logger.info("Baseline PL traffic verified") + + stop_event = threading.Event() + traffic_results = {} + traffic_thread = InterruptableThread( + target=_send_continuous_pl_traffic, + args=(ptfadapter, pl_config, recv_ports, stop_event, traffic_results), + ) + traffic_thread.start() + time.sleep(2) + + try: + logger.info(f"Step 1: Triggering planned shutdown on {repair_target} DPU") + set_dash_ha_scope( + localhost, + duthosts[repair_index], + ptfhost, + repair_vdpu_key, + "dead", + ha_owner, + disabled=True, + ) + + pytest_assert( + verify_ha_state(duthosts[repair_index], repair_vdpu_key, "dead"), + f"{repair_title} DPU did not reach dead state after planned shutdown", + ) + pytest_assert( + verify_ha_state(duthosts[peer_index], peer_vdpu_key, "standalone"), + f"{peer_title} HA state is not standalone", + ) + + logger.info( + f"{repair_title} DPU is in dead state, {peer_title} DPU is in standalone state" + ) + + logger.info( + f"Step 2: Updating HA set to replace '{repair_vdpu_key}' with '{new_vdpu_id}' " + "on all switches and DPUs" + ) + old_vdpu_id = f"vdpu{repair_index}_{selected_dpuhosts[repair_index].dpu_index}" + _update_ha_set_with_replacement_dpu( + localhost=localhost, + duthosts=duthosts, + ptfhost=ptfhost, + ha_owner=ha_owner, + old_vdpu_id=old_vdpu_id, + new_vdpu_id=new_vdpu_id, + old_duthost=duthosts[repair_index], + peer_duthost=duthosts[peer_index], + replacement_duthost=replacement_duthost, + repair_target=repair_target, + ha_set_id=ha_set_id, + ) + repair_runtime_state["replacement_scope_programmed"] = True + logger.info( + f"Replacement {repair_target} DPU scope programmed with disabled admin state; " + "HA role activation is verified in the next step" + ) + + logger.info( + f"HA: Step 3: Programming ENIs on the replacement DPU: {replacement_dpuhost.hostname}" + ) + program_eni_pl_on_dpu(localhost, ptfhost, replacement_duthost, replacement_dpuhost) + + logger.info(f"Step 4: Activating the new {repair_target} DPU") + pytest_assert( + _activate_replacement_dpu( + localhost, + replacement_duthost, + ptfhost, + new_vdpu_key, + ha_owner, + repair_target, + ), + f"Failed to activate HA on replacement {repair_target} DPU ({new_vdpu_key})", + ) + logger.info( + f"Replacement {repair_target} DPU reached {expected_replacement_state} state" + ) + + logger.info("Step 5: Verifying final HA states") + pytest_assert( + verify_ha_state( + replacement_duthost, + new_vdpu_key, + expected_replacement_state, + ), + f"Replacement {repair_target} DPU HA state is not {expected_replacement_state}", + ) + pytest_assert( + verify_ha_state(duthosts[peer_index], peer_vdpu_key, expected_peer_state), + f"{peer_title} DPU HA state is not {expected_peer_state} after {repair_target} replacement", + ) + + logger.info( + f"{repair_title} DPU replacement test completed successfully: replacement DPU " + f"'{new_vdpu_key}' is {expected_replacement_state}, surviving peer DPU " + f"'{peer_vdpu_key}' is {expected_peer_state}" + ) + finally: + stop_event.set() + traffic_thread.join(timeout=30) + if traffic_thread.is_alive(): + logger.warning("Traffic thread still running after 30 seconds; waiting 5 more seconds") + traffic_thread.join(timeout=5) + pytest_assert( + not traffic_thread.is_alive(), + "Continuous PL traffic thread did not stop cleanly", + ) + ptfadapter.dataplane.flush() + + sent = traffic_results.get("sent", 0) + received = traffic_results.get("received", 0) + loss_pct = 100 * (sent - received) / max(sent, 1) + logger.info( + f"Traffic: sent={sent} received={received} loss={sent - received} ({loss_pct:.1f}%)" + ) + assert loss_pct <= MAX_TRAFFIC_LOSS_PCT, ( + f"Traffic loss {loss_pct:.1f}% exceeds threshold " + f"{MAX_TRAFFIC_LOSS_PCT}% (sent={sent} received={received})" + ) diff --git a/tests/ha/test_ha_split_brain.py b/tests/ha/test_ha_split_brain.py index 9c911b3d1ee..cd636c06f3a 100644 --- a/tests/ha/test_ha_split_brain.py +++ b/tests/ha/test_ha_split_brain.py @@ -1,13 +1,12 @@ import logging -import configs.privatelink_config as pl import pytest import time from constants import ( NPU_DATAPLANE_PORT ) from tests.common.helpers.assertions import pytest_assert -from gnmi_utils import apply_messages +from tests.ha.conftest import apply_dash_pl_pipeline_config from ha_utils import ( set_dash_ha_scope, activate_secondary_dash_ha, @@ -53,46 +52,7 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"Start DASH config on {duthost.hostname} dpu {dpuhost.dpu_index} with {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield diff --git a/tests/ha/test_ha_steady_state_fnic.py b/tests/ha/test_ha_steady_state_fnic.py index c2e3110aa4a..37570ddf70c 100644 --- a/tests/ha/test_ha_steady_state_fnic.py +++ b/tests/ha/test_ha_steady_state_fnic.py @@ -1,6 +1,5 @@ import logging import random -import concurrent.futures import configs.privatelink_config as pl import ptf.packet as scapy @@ -8,11 +7,11 @@ import pytest from tests.common.helpers.assertions import pytest_assert from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF, REMOTE_PTF_SEND_INTF -from gnmi_utils import apply_messages from packets import inbound_pl_packets, outbound_pl_packets -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config from tests.common.dash_utils import verify_tunnel_packets from ha_dash_flow_utils import compare_flow_tables_pdsctl +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -25,11 +24,6 @@ NUM_PACKETS = 5 -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - def _build_fnic_pkt_set(config, encap_proto, ptfadapter): """Build a list of NUM_PACKETS bidirectional fnic packet tuples for a given DPU config.""" pkt_sets = [] @@ -66,55 +60,10 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_FNIC_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.ROUTING_TYPE_VNET_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG, - **pl.TUNNEL1_CONFIG, - } - logger.info(f"configure on {duthost.hostname} dpu {dpuhost.dpu_index} {base_config_messages}") - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_VNET_MAPPING_CONFIG, - **pl.VM_SUBNET_ROUTE_WITH_TUNNEL_SINGLE_ENDPOINT, - } - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - if 'pensando' not in dpuhost.facts['asic_type']: - route_rule_messages = { - **pl.VM_VNI_ROUTE_RULE_CONFIG, - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG, - **pl.TRUSTED_VNI_ROUTE_RULE_CONFIG, - } - logger.info(route_rule_messages) - apply_messages(localhost, duthost, ptfhost, route_rule_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_FNIC_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_FNIC_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost, floating_nic=True) yield - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) @pytest.mark.parametrize("encap_proto", ["vxlan", "gre"]) diff --git a/tests/ha/test_ha_steady_state_pl.py b/tests/ha/test_ha_steady_state_pl.py index 452fc918610..5be760544a1 100644 --- a/tests/ha/test_ha_steady_state_pl.py +++ b/tests/ha/test_ha_steady_state_pl.py @@ -1,15 +1,15 @@ import logging -import configs.privatelink_config as pl import ptf.testutils as testutils import pytest -import concurrent.futures +from configs.privatelink_config import APPLIANCE_VIP from tests.common.helpers.assertions import pytest_assert from constants import LOCAL_PTF_INTF, REMOTE_PTF_RECV_INTF, REMOTE_PTF_SEND_INTF -from gnmi_utils import apply_messages from packets import outbound_pl_packets, inbound_pl_packets -from tests.common.config_reload import config_reload +from tests.ha.conftest import apply_dash_pl_pipeline_config +from ha_bgp_utils import check_vip_advertised_to_t2 from ha_dash_flow_utils import compare_flow_tables +from ha_utils import parallel_config_reload_dpuhosts logger = logging.getLogger(__name__) @@ -24,11 +24,6 @@ """ -def reload_config_for_host(dpuhost): - logger.info(f"config reload on {dpuhost.hostname}") - config_reload(dpuhost, safe_reload=True, yang_validate=False) - - @pytest.fixture(autouse=True, scope="module") def common_setup_teardown( localhost, @@ -46,57 +41,18 @@ def common_setup_teardown( if skip_config: return - for i in range(len(duthosts)): - duthost = duthosts[i] - dpuhost = dpuhosts[i] - base_config_messages = { - **pl.APPLIANCE_CONFIG, - **pl.ROUTING_TYPE_PL_CONFIG, - **pl.VNET_CONFIG, - **pl.ROUTE_GROUP1_CONFIG, - **pl.METER_POLICY_V4_CONFIG - } - logger.info(f"configure on {duthost.hostname} dpu {dpuhost.dpu_index} {base_config_messages}") - - apply_messages(localhost, duthost, ptfhost, base_config_messages, dpuhost.dpu_index) - - route_and_mapping_messages = { - **pl.PE_VNET_MAPPING_CONFIG, - **pl.PE_SUBNET_ROUTE_CONFIG, - **pl.VM_SUBNET_ROUTE_CONFIG - } - - if 'bluefield' in dpuhost.facts['asic_type']: - route_and_mapping_messages.update({ - **pl.INBOUND_VNI_ROUTE_RULE_CONFIG - }) - - logger.info(route_and_mapping_messages) - apply_messages(localhost, duthost, ptfhost, route_and_mapping_messages, dpuhost.dpu_index) - - meter_rule_messages = { - **pl.METER_RULE1_V4_CONFIG, - **pl.METER_RULE2_V4_CONFIG, - } - logger.info(meter_rule_messages) - apply_messages(localhost, duthost, ptfhost, meter_rule_messages, dpuhost.dpu_index) - - logger.info(pl.ENI_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_CONFIG, dpuhost.dpu_index) - - logger.info(pl.ENI_ROUTE_GROUP1_CONFIG) - apply_messages(localhost, duthost, ptfhost, pl.ENI_ROUTE_GROUP1_CONFIG, dpuhost.dpu_index) + apply_dash_pl_pipeline_config(localhost, duthosts, dpuhosts, ptfhost) yield - with concurrent.futures.ThreadPoolExecutor(max_workers=len(dpuhosts)) as executor: - # Map the reload_config_for_host function to the dpuhosts list - executor.map(reload_config_for_host, dpuhosts) + parallel_config_reload_dpuhosts(dpuhosts) @pytest.mark.parametrize("encap_proto", ["vxlan", "gre"]) def test_privatelink_basic_transform( ptfadapter, + duthosts, dpuhosts, + nbrhosts, activate_dash_ha_from_json, ha_owner, dash_pl_config, @@ -125,3 +81,5 @@ def test_privatelink_basic_transform( pytest_assert(flow_op, "Expected identical flow tables on primary and standby") testutils.send(ptfadapter, dash_pl_config[1][REMOTE_PTF_SEND_INTF], pe_to_dpu_pkt, 1) testutils.verify_packet(ptfadapter, exp_dpu_to_vm_pkt, dash_pl_config[0][LOCAL_PTF_INTF]) + + check_vip_advertised_to_t2(duthosts, APPLIANCE_VIP) diff --git a/tests/high_frequency_telemetry/utilities.py b/tests/high_frequency_telemetry/utilities.py index 42487349e04..726f9d98f6a 100644 --- a/tests/high_frequency_telemetry/utilities.py +++ b/tests/high_frequency_telemetry/utilities.py @@ -962,14 +962,14 @@ def validate_enabled_stream_output( f"Successfully verified {len(counter_matches)} counter values " f"are > {min_counter_value}") - # Validate Msg/s if poll_interval is provided + # Always parse Msg/s so callers that only need to detect active streams can + # inspect the values without requesting strict rate validation. msg_per_sec_matches = [] msg_validation_result = None + msg_pattern = r'Msg/s:\s+(\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)' + msg_per_sec_matches = re.findall(msg_pattern, stable_output) if expected_poll_interval: - msg_pattern = r'Msg/s:\s+(\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)' - msg_per_sec_matches = re.findall(msg_pattern, stable_output) - if msg_per_sec_matches: msg_values = [float(m) for m in msg_per_sec_matches] diff --git a/tests/layer1/test_fec_error.py b/tests/layer1/test_fec_error.py index 01bc1d4f7bf..78b2681ba97 100644 --- a/tests/layer1/test_fec_error.py +++ b/tests/layer1/test_fec_error.py @@ -83,14 +83,15 @@ def validate_predicted_flr(value_string) -> bool: Expected format : * 0 * 7.81e-10 (89%) + * 3.38e+00 (nan%) """ # Pattern for just "0" if value_string == "0": return True # Pattern for scientific notation with required accuracy percentage - # e.g., "7.81e-10 (89%)" - pattern = r'^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)\s+\(\d+%\)$' + # e.g., "7.81e-10 (89%)" or "3.38e+00 (nan%)" + pattern = r'^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)\s+\((\d+|nan)%\)$' if re.match(pattern, value_string): return True diff --git a/tests/live_addon_docker/__init__.py b/tests/live_addon_docker/__init__.py new file mode 100644 index 00000000000..e3f558a0be9 --- /dev/null +++ b/tests/live_addon_docker/__init__.py @@ -0,0 +1 @@ +# Live-addon docker install and validation tests (vendor-specific JSON under files/) diff --git a/tests/live_addon_docker/conftest.py b/tests/live_addon_docker/conftest.py new file mode 100644 index 00000000000..329718c0fa3 --- /dev/null +++ b/tests/live_addon_docker/conftest.py @@ -0,0 +1,154 @@ +""" +Live-addon docker tests load a single JSON that defines docker_run, health, and validation. + +Default path: ``files/_live_addon_docker.json`` (``asic_type`` from the enum DUT). +Override with ``--live-addon-docker-config``. +Commands on the DUT are assembled in live_addon_docker_helpers from that JSON only. + +Registry and tag at test time: ``--live_addon_docker_registry``, ``--live_addon_docker_image_tag``. +""" + +import logging +import os + +import pytest + +from tests.live_addon_docker import live_addon_docker_helpers as lad + +logger = logging.getLogger(__name__) + + +def _cli_image_tag(request): + val = request.config.getoption("--live_addon_docker_image_tag", default=None) + if val and str(val).strip(): + return str(val).strip() + return None + + +def pytest_addoption(parser): + parser.addoption( + "--live-addon-docker-config", + action="store", + default=None, + help=( + "Path to live-addon docker JSON (default: files/_live_addon_docker.json " + "from DUT facts; fails if missing)" + ), + ) + parser.addoption( + "--live-addon-docker-tarball", + action="store", + default=None, + help="Full path to the .gz image on the test runner (default: tarball_filename from JSON)", + ) + parser.addoption( + "--live_addon_docker_registry", + action="store", + default=None, + help=( + "Docker registry host for live-addon image pull (e.g. myacr.azurecr.io). " + "Overrides Ansible creds docker_registry_host for this test module only. " + "Use --public_docker_registry for no-login public host (same as swap_syncd)." + ), + ) + parser.addoption( + "--live_addon_docker_image_tag", + action="store", + default=None, + help=( + "Docker image tag for registry pull and docker_run.image_ref (overrides JSON tag and " + "dut os_version for pull). For CI builds, e.g. kube-20260527-202505-amd64." + ), + ) + + +@pytest.fixture(scope="module") +def live_addon_docker_vendor_cfg(request, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + opt = request.config.getoption("--live-addon-docker-config") + if opt: + path = os.path.abspath(opt) + else: + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + asic_type = (duthost.facts.get("asic_type") or "").strip() + if not asic_type: + pytest.fail("DUT asic_type is empty; cannot resolve live-addon docker JSON path") + path = lad.default_live_addon_config_path(asic_type) + logger.info( + "live_addon_docker_vendor_cfg: using default from asic_type=%s -> %s", + asic_type, + path, + ) + if not os.path.isfile(path): + pytest.skip("Live-addon docker config not found: {}".format(path)) + cfg = lad.load_live_addon_config(path) + tag = _cli_image_tag(request) + if tag: + cfg = lad.apply_image_tag_to_config(cfg, tag) + logger.info( + "live_addon_docker_vendor_cfg: image tag overridden by CLI -> %s", + cfg["docker_run"]["image_ref"], + ) + return cfg + + +@pytest.fixture(scope="module") +def live_addon_docker_local_tarball_optional(request, live_addon_docker_vendor_cfg): + """ + Path to .gz on the test runner if that file exists; otherwise None. + Resolution on the DUT (image already loaded vs ~/ vs copy) is done in + live_addon_docker_install_source — same idea as swap_syncd using local docker images. + """ + override = request.config.getoption("--live-addon-docker-tarball") + local_path = lad.resolve_local_tarball_path(live_addon_docker_vendor_cfg, lad.MODULE_DIR, override) + if os.path.isfile(local_path): + logger.info("Live-addon docker tarball on test runner: %s", local_path) + return local_path + logger.info( + "No live-addon tarball on test runner at %s — will use image or tarball on DUT if present", + local_path, + ) + return None + + +@pytest.fixture(scope="module") +def live_addon_docker_install_source( + request, + duthosts, + enum_rand_one_per_hwsku_frontend_hostname, + live_addon_docker_vendor_cfg, + live_addon_docker_local_tarball_optional, +): + """ + Decide install source (priority; registry first when configured, same Ansible creds as swap_syncd): + 1) registry pull on DUT (docker pull + tag); use --public_docker_registry like swap_syncd for public host + 2) tarball under admin home on DUT (~/tarball_filename) + 3) tarball on test runner (copy to /tmp on DUT) + 4) image already on DUT (docker image inspect) + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + public_reg = request.config.getoption("--public_docker_registry") + registry_override = request.config.getoption("--live_addon_docker_registry") + res = lad.resolve_live_addon_install_source( + duthost, + live_addon_docker_vendor_cfg, + live_addon_docker_local_tarball_optional, + public_docker_registry=public_reg, + docker_registry_host_override=registry_override, + registry_image_tag=_cli_image_tag(request), + ) + if res.kind == "none": + pytest.skip( + "Live-addon docker not available: no matching image on DUT, no tarball at {0}, " + "and no tarball on the test runner. Pre-load the image (`docker load`), copy " + "{1} to the DUT home dir, place {1} under tests/live_addon_docker/ on the test " + "runner, pass --live-addon-docker-tarball, or ensure docker_registry_host in Ansible creds " + "for registry pull (or omit docker_registry_host so registry is skipped and use a tarball / local image)." + .format( + lad.dut_home_tarball_path(live_addon_docker_vendor_cfg), + live_addon_docker_vendor_cfg["tarball_filename"], + ) + ) + if res.kind == "runner_tarball": + duthost.copy(src=live_addon_docker_local_tarball_optional, dest=res.remote_tarball_path) + logger.info("live_addon_docker_install_source: kind=%s", res.kind) + yield duthost, res diff --git a/tests/live_addon_docker/files/cisco-8000_live_addon_docker.json b/tests/live_addon_docker/files/cisco-8000_live_addon_docker.json new file mode 100644 index 00000000000..25a45c85181 --- /dev/null +++ b/tests/live_addon_docker/files/cisco-8000_live_addon_docker.json @@ -0,0 +1,68 @@ +{ + "vendor": "cisco", + "description": "Cisco 8000 live-addon docker. Image docker-live-addon-cisco; container cisco-utility.", + "tarball_filename": "docker-live-addon-cisco.gz", + "dut_tarball_home": "/home/admin", + "candidate_image_refs": [], + "version_matrix": [ + { + "utility_package_version_glob": "202405*", + "compatible_sonic_globs": ["202411*", "202505*"] + }, + { + "utility_package_version_glob": "202411*", + "compatible_sonic_globs": ["202411*", "202505*"] + } + ], + "docker_run": { + "image_ref": "docker-live-addon-cisco:latest", + "container_name": "cisco-utility", + "cli_args": [ + "-t", + "--privileged", + "--pid=host", + "-v", "/:/hostroot:rw", + "-v", "/opt/cisco:/opt/cisco", + "--tmpfs", "/tmp/", + "--net=host", + "--log-opt", "max-size=2M", + "--log-opt", "max-file=5" + ] + }, + "health": { + "port": 50200, + "url_path": "/health", + "bind_host": "127.0.0.1", + "expect_http_code": 200, + "wait_seconds_before_check": 0, + "probe_interval_seconds": 30, + "probe_timeout_seconds": 900 + }, + "validation": { + "docker_container_name": "cisco-utility", + "expected_processes": [ + "start", + "health-monitor", + "health-server" + ], + "startup_log": { + "wait_seconds": 900, + "poll_interval_seconds": 30, + "session_start_pattern": "supervisord started with pid", + "required_patterns": [ + "supervisord started with pid", + "success: start entered RUNNING state", + "success: health-monitor entered RUNNING state", + "success: health-server entered RUNNING state", + "diagnostic is running now" + ], + "forbidden_patterns": [ + "spawnerr", + "exited too quickly", + "entered FATAL state", + "BACKOFF", + "[FAILED]" + ] + } + } +} diff --git a/tests/live_addon_docker/live_addon_docker_helpers.py b/tests/live_addon_docker/live_addon_docker_helpers.py new file mode 100644 index 00000000000..e39332f0095 --- /dev/null +++ b/tests/live_addon_docker/live_addon_docker_helpers.py @@ -0,0 +1,1382 @@ +"""Helpers for loading, installing, and validating live-addon docker images on the DUT. + +Commands run on the DUT are built from the vendor JSON (see ``files/*.json``). +Default path: ``files/_live_addon_docker.json`` (from DUT ``facts['asic_type']``), +for example ``files/_live_addon_docker.json``. Override with ``--live-addon-docker-config``. + +The JSON must define ``vendor``, ``docker_run`` (``docker load`` if needed, then ``docker run``), +``health``, and ``validation`` (container name for checks). ``docker_run.image_ref`` is derived as +``docker-live-addon-[:tag]`` unless set explicitly (must match ACR repo for registry pull). +Optional fields: ``tarball_filename``, ``version_matrix``, ``candidate_image_refs``. +Registry pull uses Ansible ``docker_registry_*`` or pytest ``--live_addon_docker_registry``; +``--live_addon_docker_image_tag`` overrides pull/run tag. ``--public_docker_registry`` for public host. +Optional ``version_matrix`` skips when live-addon vs DUT SONiC is not declared compatible +(see ``require_version_matrix_or_skip``). +""" + +import collections +import copy +import fnmatch +import json +import logging +import os +import re +import shlex +import time + +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.dut_utils import creds_on_dut, is_container_running +from tests.common.system_utils.docker import download_image, load_docker_registry_info +from tests.common.utilities import wait_until + +logger = logging.getLogger(__name__) + +MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def default_live_addon_config_path(asic_type): + """ + Return absolute path to ``files/_live_addon_docker.json`` under this package. + + ``asic_type`` is ``duthost.facts['asic_type']`` (selects ``files/_live_addon_docker.json``). + """ + name = (asic_type or "").strip() + if not name: + raise ValueError("asic_type is empty") + return os.path.abspath(os.path.join(MODULE_DIR, "files", "{}_live_addon_docker.json".format(name))) + + +# SONiC DUT default login home (same idea as admin $HOME for pre-staged tarballs) +DUT_ADMIN_HOME = "/home/admin" + +# Post-teardown checks (always on; not configurable via vendor JSON) +_DEFAULT_SYSLOG_TAIL_LINES = 400 +_DOCKER_RUN_UP_TIMEOUT = 120 +_DOCKER_RUN_UP_INTERVAL = 3 +_DEFAULT_SYSLOG_ERROR_PATTERN = ( + "(segfault|SIGSEGV|SIGABRT|Out of memory|oom-kill|FATAL|panic)" +) + +# Default supervisord-managed programs (names from ``supervisorctl status`` in live-addon container). +DEFAULT_LIVE_ADDON_EXPECTED_PROCESSES = ( + "start", + "health-monitor", + "health-server", +) + + +def expected_processes_from_validation(val): + """Return ``validation.expected_processes`` or the default list; empty list means skip process checks.""" + if not val: + return list(DEFAULT_LIVE_ADDON_EXPECTED_PROCESSES) + expected = val.get("expected_processes") + if expected is None: + return list(DEFAULT_LIVE_ADDON_EXPECTED_PROCESSES) + return expected + + +# Generic poll timing for ``validation.startup_log`` (vendor JSON overrides ``wait_seconds``). +# How long supervisord/diagnostic lines take depends on the vendor; set ``wait_seconds`` per vendor JSON. +DEFAULT_STARTUP_LOG_WAIT_SECONDS = 120 +DEFAULT_STARTUP_LOG_POLL_INTERVAL_SECONDS = 30 +# supervisord ``RUNNING`` poll only (does not use startup_log or health ``probe_timeout_seconds``). +DEFAULT_PROCESS_WAIT_SECONDS = 120 +DEFAULT_PROCESS_POLL_INTERVAL_SECONDS = 30 +_STARTUP_LOG_FAILURE_SNIPPET_CHARS = 6000 + +# ``docker inspect -f`` Go template for current container start time (used with ``docker logs --since``). +_DOCKER_INSPECT_STARTED_AT_FMT = "{{.State.StartedAt}}" +_STARTED_AT_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}") + +# ACR/docker repository name: docker-live-addon- (from JSON ``vendor`` field). +LIVE_ADDON_IMAGE_REPO_PREFIX = "docker-live-addon" +DEFAULT_LIVE_ADDON_IMAGE_TAG = "latest" + +InstallSource = collections.namedtuple( + "InstallSource", ["kind", "remote_tarball_path", "image_ref"] +) +# kind: "image_present" | "dut_home_tarball" | "runner_tarball" | "none" +# Registry installs use kind "image_present" after pull+tag (see try_registry_pull_live_addon_image). + + +def _apply_live_addon_registry_creds(creds, public_docker_registry=False, docker_registry_host_override=None): + """Update *creds* in place for live-addon registry pull. Do not log *creds* (contains secrets).""" + if public_docker_registry: + creds["docker_registry_host"] = (creds.get("public_docker_registry_host") or "").strip() + creds["docker_registry_username"] = "" + creds["docker_registry_password"] = "" + override = (docker_registry_host_override or "").strip() + if override: + creds["docker_registry_host"] = override + return creds + + +def live_addon_image_repository(vendor): + """ + Return the live-addon image repository name for a vendor (``docker-live-addon-``). + + ``vendor`` comes from the top-level ``vendor`` field in the vendor JSON. + """ + name = (vendor or "").strip().lower() + if not name: + raise ValueError("vendor config must set 'vendor' for live-addon image repository name") + return "{}-{}".format(LIVE_ADDON_IMAGE_REPO_PREFIX, name) + + +def resolve_docker_run_image_ref(cfg): + """ + Build ``docker_run.image_ref`` from ``vendor`` and optional ``docker_run.image_tag``. + + Repository is always ``docker-live-addon-``. Tag defaults to ``latest``; an explicit + ``docker_run.image_tag`` or legacy ``docker_run.image_ref`` (tag portion only) overrides it. + """ + dr = cfg.get("docker_run") or {} + tag = dr.get("image_tag") + if tag is None and dr.get("image_ref"): + tag = image_ref_to_tag(dr["image_ref"]) + if not tag: + tag = DEFAULT_LIVE_ADDON_IMAGE_TAG + return "{}:{}".format(live_addon_image_repository(cfg.get("vendor")), str(tag).strip()) + + +def normalize_live_addon_config(cfg): + """Set ``docker_run.image_ref`` from ``vendor`` (and optional tag) after loading JSON.""" + cfg.setdefault("docker_run", {}) + dr = cfg["docker_run"] + explicit_ref = (dr.get("image_ref") or "").strip() + if explicit_ref: + repo = _image_repository_from_image_ref(explicit_ref) + expected_repo = live_addon_image_repository(cfg.get("vendor")) + if repo and repo != expected_repo: + logger.warning( + "docker_run.image_ref repository %r does not match vendor-derived %r; using vendor repo", + repo, + expected_repo, + ) + resolved = resolve_docker_run_image_ref(cfg) + dr["image_ref"] = resolved + logger.info("Resolved docker_run.image_ref: %s", resolved) + return cfg + + +def apply_image_tag_to_config(cfg, image_tag): + """Return config copy with ``docker_run.image_tag`` set and ``image_ref`` re-resolved.""" + tag = (image_tag or "").strip() + if not tag: + return cfg + out = copy.deepcopy(cfg) + out.setdefault("docker_run", {}) + out["docker_run"]["image_tag"] = tag + return normalize_live_addon_config(out) + + +def load_live_addon_config(config_path): + """Load vendor JSON parameters and resolve ``docker_run.image_ref`` from ``vendor``.""" + with open(config_path, encoding="utf-8") as handle: + cfg = json.load(handle) + return normalize_live_addon_config(cfg) + + +def image_ref_to_tag(image_ref): + """Return the tag or digest label after ':' in a docker image ref, or 'latest' if missing.""" + if not image_ref or not isinstance(image_ref, str): + return "latest" + ref = image_ref.strip() + if ":" in ref: + return ref.split(":")[-1].strip() + return "latest" + + +def live_addon_image_tag_for_matrix(cfg, install_source): + """ + Docker image ref tag (part after ``:``), for logging / resolution only. + + ``version_matrix`` uses ``package.version`` from image metadata, not this tag. + + If the image is already on the DUT, use that ref's tag. Otherwise use docker_run.image_ref + (expected tag after docker load). + """ + if install_source.kind == "image_present" and install_source.image_ref: + return image_ref_to_tag(install_source.image_ref) + dr = cfg.get("docker_run") or {} + return image_ref_to_tag(dr.get("image_ref", "")) + + +def sonic_matches_sonic_glob(duthost, glob_pattern): + """ + True if DUT ``os_version``, ``sonic_release``, or ``show version`` one-liner matches glob. + + Trains like ``202411``, ``202505`` appear in ``os_version`` strings such as + ``SONiC._202411.-...`` or ``202411.1.2.3.4``; globs like ``202411*`` + match the full ``os_version`` line. + """ + if fnmatch.fnmatch(duthost.os_version, glob_pattern): + return True + sr = getattr(duthost, "sonic_release", None) + if sr and fnmatch.fnmatch(str(sr), glob_pattern): + return True + try: + ver_line = duthost.shell("show version | head -1", module_ignore_errors=True).get("stdout", "") + if ver_line.strip() and fnmatch.fnmatch(ver_line.strip(), glob_pattern): + return True + except Exception as exc: + logger.warning( + "sonic_matches_sonic_glob: optional show version line probe failed (ignored): %s", + exc, + ) + return False + + +def get_docker_image_config_labels(duthost, image_ref): + """ + Return ``Config.Labels`` from ``docker image inspect`` (full JSON, no Go ``-f`` templates). + + Some live-addon images set ``com.azure.sonic.manifest`` (JSON string) and ``Tag`` labels. + """ + if not image_ref or not str(image_ref).strip(): + return {} + qref = shlex.quote(str(image_ref).strip()) + out = duthost.shell("sudo docker image inspect {}".format(qref), module_ignore_errors=True) + if out.get("rc") != 0: + logger.warning("docker image inspect failed for %s: %s", image_ref, out.get("stderr", "")) + return {} + try: + data = json.loads(out["stdout"]) + if not data: + return {} + return data[0].get("Config", {}).get("Labels") or {} + except (ValueError, TypeError, KeyError, IndexError) as exc: + logger.warning("Could not parse docker image inspect JSON for %s: %s", image_ref, exc) + return {} + + +def package_version_from_azure_sonic_manifest_labels(labels): + """ + Parse ``package.version`` from label ``com.azure.sonic.manifest`` (JSON), e.g. ``202405.1.0-0``. + """ + if not labels: + return None + raw = labels.get("com.azure.sonic.manifest") + if not raw or not isinstance(raw, str): + return None + try: + manifest = json.loads(raw) + pkg = manifest.get("package") or {} + ver = pkg.get("version") + return str(ver).strip() if ver is not None else None + except (ValueError, TypeError, AttributeError): + return None + + +def tag_label_from_image_labels(labels): + """Optional ``Tag`` label on the image (e.g. build id string).""" + if not labels: + return None + t = labels.get("Tag") + return str(t).strip() if t else None + + +def _version_matrix_row_matches_utility(row, package_version): + """ + Row may filter by ``utility_image_version_glob`` and/or ``utility_package_version_glob``. + + Both keys apply to ``package.version`` parsed from label ``com.azure.sonic.manifest`` (not the + Docker image ``:tag``). If a key is set but ``package_version`` is missing, the row does not + match. + """ + if "utility_image_version_glob" in row and row["utility_image_version_glob"] is not None: + if not package_version: + return False + if not fnmatch.fnmatch(package_version, row["utility_image_version_glob"]): + return False + if "utility_package_version_glob" in row and row["utility_package_version_glob"] is not None: + if not package_version: + return False + if not fnmatch.fnmatch(package_version, row["utility_package_version_glob"]): + return False + return True + + +def require_version_matrix_or_skip(duthost, cfg, resolved_image_ref): + """ + Optional JSON ``version_matrix``: skip when live-addon ``package.version`` (image metadata) and + DUT SONiC build are not declared compatible. + + Call **after** the image exists on the DUT (``docker load`` / present) and **before** + ``docker run``, passing ``resolved_image_ref`` from ``docker_run.image_ref``. + + Omitted, null, or ``[]`` disables this check. + """ + matrix = cfg.get("version_matrix") + if not matrix: + return + + labels = get_docker_image_config_labels(duthost, resolved_image_ref) + utility_tag = image_ref_to_tag(resolved_image_ref) + package_ver = package_version_from_azure_sonic_manifest_labels(labels) + + matching_rows = [] + for row in matrix: + if _version_matrix_row_matches_utility(row, package_ver): + matching_rows.append(row) + + if not matching_rows: + pytest.skip( + "version_matrix: no row matches package.version={!r} (image ref {!r}, ref tag={!r}, " + "labels Tag={!r}).".format( + package_ver, + resolved_image_ref, + utility_tag, + tag_label_from_image_labels(labels), + ) + ) + + allowed = [] + for row in matching_rows: + allowed.extend(row.get("compatible_sonic_globs") or []) + + if not allowed: + pytest.skip( + "version_matrix: matching row has no compatible_sonic_globs (package.version={!r})".format( + package_ver + ) + ) + + if any(sonic_matches_sonic_glob(duthost, g) for g in allowed): + return + + pytest.skip( + "version_matrix: DUT SONiC not compatible: os_version={!r} sonic_release={!r}; " + "package.version={!r} (ref tag={!r}); allowed sonic globs: {}".format( + duthost.os_version, + getattr(duthost, "sonic_release", ""), + package_ver, + utility_tag, + allowed, + ) + ) + + +def _image_refs_to_try(cfg): + """Names/tags to match swap_syncd-style 'already on DUT' behavior (docker image inspect).""" + refs = [] + dr = cfg.get("docker_run") or {} + if dr.get("image_ref"): + refs.append(dr["image_ref"].strip()) + for extra in cfg.get("candidate_image_refs", []): + ex = extra.strip() + if ex and ex not in refs: + refs.append(ex) + return refs + + +def find_existing_live_addon_image(duthost, cfg): + """ + Return first image ref that exists in local docker storage on the DUT, or None. + Same idea as swap_syncd checking `docker image inspect docker-syncd--rpc`. + """ + for ref in _image_refs_to_try(cfg): + if image_exists(duthost, ref): + logger.info("Found existing live-addon image on DUT: %s", ref) + return ref + return None + + +def dut_home_tarball_path(cfg): + """Path under admin home for a pre-copied .gz (``~/`` + ``tarball_filename`` from JSON).""" + name = cfg.get("tarball_filename") + if not name: + raise ValueError("vendor config must set tarball_filename") + home = cfg.get("dut_tarball_home", DUT_ADMIN_HOME) + return os.path.join(home, name) + + +def dut_file_exists(duthost, path): + return duthost.command("sudo test -f {}".format(path), module_ignore_errors=True)["rc"] == 0 + + +def _image_repository_from_image_ref(image_ref): + """Return repository part before the last ':' in ``name:tag``; ``image_ref`` if no colon.""" + ref = (image_ref or "").strip() + if not ref: + return None + pos = ref.rfind(":") + if pos <= 0: + return ref + return ref[:pos].strip() + + +def _live_addon_registry_pull_settings(cfg, registry_image_tag=None): + """ + Return settings for live-addon ``docker pull``. + + Repository from ``docker_run.image_ref`` (``docker-live-addon-``). Tag is + ``registry_image_tag`` when set (CLI), else tag from ``image_ref`` if not ``latest``, + else ``duthost.os_version`` at pull time. + """ + dr = cfg.get("docker_run") or {} + target_ref = (dr.get("image_ref") or "").strip() + if not target_ref: + return None + + image_name = _image_repository_from_image_ref(target_ref) + if not image_name: + logger.warning("live-addon registry pull: cannot parse repository from docker_run.image_ref") + return None + + if registry_image_tag and str(registry_image_tag).strip(): + image_version = str(registry_image_tag).strip() + else: + ref_tag = image_ref_to_tag(target_ref) + image_version = None if ref_tag == DEFAULT_LIVE_ADDON_IMAGE_TAG else ref_tag + + return {"image_name": image_name, "image_version": image_version, "target_ref": target_ref} + + +def resolve_live_addon_install_source( + duthost, + cfg, + local_runner_tarball_path, + public_docker_registry=False, + docker_registry_host_override=None, + registry_image_tag=None, +): + """ + Resolve where to get the image from. + + **Registry is tried first** when ``docker_run.image_ref`` is set and Ansible defines + ``docker_registry_host`` (after applying ``public_docker_registry`` the same way as + ``swap_syncd``: host becomes ``public_docker_registry_host``, username/password cleared). + Image name is derived from ``docker_run.image_ref``, tag from ``duthost.os_version``. On failure + or missing registry host, resolution continues with DUT tarball, runner tarball, then an image + already in docker storage. + + 1) ``docker pull`` + ``docker tag`` to ``docker_run.image_ref`` (when registry path active). + 2) Tarball under admin home on DUT — ``docker load -i``. + 3) Tarball on the ansible test runner — copy to /tmp on DUT, then ``docker load -i``. + 4) Image already on DUT (docker image inspect). + + If none apply, returns InstallSource(kind='none', ...). + """ + reg_settings = _live_addon_registry_pull_settings(cfg, registry_image_tag=registry_image_tag) + if reg_settings is not None: + ref = try_registry_pull_live_addon_image( + duthost, + reg_settings, + public_docker_registry=public_docker_registry, + docker_registry_host_override=docker_registry_host_override, + ) + if ref: + logger.info("Live-addon docker image from registry: %s", ref) + return InstallSource("image_present", None, ref) + logger.info( + "Live-addon registry pull did not produce an image; " + "trying DUT tarball, runner tarball, local image" + ) + + dut_path = dut_home_tarball_path(cfg) + if dut_file_exists(duthost, dut_path): + logger.info("Live-addon docker tarball on DUT (will docker load): %s", dut_path) + return InstallSource("dut_home_tarball", dut_path, None) + + if local_runner_tarball_path and os.path.isfile(local_runner_tarball_path): + base = os.path.basename(local_runner_tarball_path) + remote = "/tmp/{}".format(base) + logger.info("Live-addon docker tarball on test runner (will copy to DUT then docker load): %s", remote) + return InstallSource("runner_tarball", remote, None) + + ref = find_existing_live_addon_image(duthost, cfg) + if ref: + logger.info("Live-addon docker image already on DUT (no docker load): %s", ref) + return InstallSource("image_present", None, ref) + + return InstallSource("none", None, None) + + +def resolve_local_tarball_path(config, search_dir, tarball_override): + """ + Resolve path to the .gz image on the ansible test server (before copy to DUT). + + Search order: + 1) tarball_override (pytest --live-addon-docker-tarball) + 2) search_dir / config['tarball_filename'] (default search_dir is this test module directory) + """ + if tarball_override: + return os.path.abspath(tarball_override) + name = config.get("tarball_filename") + if not name: + raise ValueError("vendor config must set tarball_filename") + return os.path.abspath(os.path.join(search_dir, name)) + + +def try_registry_pull_live_addon_image( + duthost, settings, public_docker_registry=False, docker_registry_host_override=None +): + """ + Pull ``{registry}/{image_name}:{image_version}`` on the DUT (Ansible ``creds`` / registry same + as ``swap_syncd`` / ``download_image``), then ``docker tag`` to ``settings['target_ref']`` + when the pulled ref differs. + + When ``public_docker_registry`` is true, applies the same credential override as the + ``swap_syncd`` fixture in ``tests/conftest.py`` (``docker_registry_host`` from + ``public_docker_registry_host``, clear username/password). + + ``settings`` comes from ``_live_addon_registry_pull_settings``. Returns ``target_ref`` on success, + or None on failure (caller tries tarballs / local image). + """ + image_name = settings["image_name"] + target_ref = settings["target_ref"] + + creds = copy.deepcopy(creds_on_dut(duthost)) + _apply_live_addon_registry_creds( + creds, + public_docker_registry=public_docker_registry, + docker_registry_host_override=docker_registry_host_override, + ) + try: + registry = load_docker_registry_info(duthost, creds) + except ValueError as exc: + logger.warning("live-addon registry pull: %s", exc) + return None + + ver = settings.get("image_version") + if ver is not None and str(ver).strip(): + image_version = str(ver).strip() + else: + image_version = duthost.os_version + + try: + download_image(duthost, registry, image_name, image_version) + except RuntimeError as exc: + logger.warning("live-addon registry pull: download failed: %s", exc) + return None + + source_ref = "{}/{}:{}".format(registry.host, image_name, image_version) + if source_ref != target_ref: + duthost.command( + "docker tag {} {}".format(shlex.quote(source_ref), shlex.quote(target_ref)) + ) + if not image_exists(duthost, target_ref): + logger.warning("live-addon registry pull: target image %r not present after pull/tag", target_ref) + return None + return target_ref + + +def build_docker_load_command(remote_tarball): + """``docker load`` line; ``remote_tarball`` is path on the DUT.""" + return "sudo docker load -i {}".format(remote_tarball) + + +def build_docker_run_command(cfg): + """ + Full ``docker run`` command from ``cfg['docker_run']``. + + Uses ``detach`` (default true) -> ``-d``, then ``cli_args``, then + ``--name ``. + """ + dr = cfg["docker_run"] + image_ref = dr["image_ref"] + name = dr["container_name"] + args = dr.get("cli_args", []) + if not isinstance(args, list): + raise ValueError("docker_run.cli_args must be a list of argv tokens") + parts = [] + if dr.get("sudo", True): + parts.append("sudo") + parts.extend(["docker", "run"]) + if dr.get("detach", True): + parts.append("-d") + parts.extend(args) + parts.extend(["--name", name, image_ref]) + return " ".join(parts) + + +def parse_image_from_docker_load(load_output): + """ + Try to extract name:tag from `docker load` stdout/stderr. + Example: 'Loaded image: docker-live-addon-:' + """ + match = re.search(r"Loaded image:\s*(\S+)", load_output) + if match: + return match.group(1).strip() + return None + + +def docker_load(duthost, remote_tarball): + """ + Run ``sudo docker load -i `` on the DUT. Fails the ansible command if load fails. + Returns combined stdout+stderr for parsing (some docker builds log ``Loaded image`` on stderr). + """ + cmd = build_docker_load_command(remote_tarball) + logger.info("Running: %s", cmd) + result = duthost.command(cmd) + out = (result.get("stdout") or "").strip() + err = (result.get("stderr") or "").strip() + combined = (out + "\n" + err).strip() + logger.info("docker load stdout: %s", out) + if err: + logger.info("docker load stderr: %s", err) + return combined + + +def docker_run_manual(duthost, cfg): + """Run container using ``build_docker_run_command(cfg)`` (all options from vendor JSON).""" + cmd = build_docker_run_command(cfg) + logger.info("Running: %s", cmd) + duthost.command(cmd) + cname = (cfg.get("docker_run") or {}).get("container_name") + if not cname: + return + + def _running(): + return is_container_running(duthost, cname) + + if wait_until(_DOCKER_RUN_UP_TIMEOUT, _DOCKER_RUN_UP_INTERVAL, 0, _running): + return + + ps_out = duthost.command( + "sudo docker ps -a --filter name={} --no-trunc".format(cname), + module_ignore_errors=True, + ).get("stdout", "") + log_out = duthost.command( + "sudo docker logs --tail 120 {}".format(cname), + module_ignore_errors=True, + ) + log_out = _command_stdout_stderr(log_out) + pytest_assert( + False, + "Container {!r} did not stay running after docker run (waited {}s). " + "docker ps -a:\n{}\n\ndocker logs:\n{}".format( + cname, _DOCKER_RUN_UP_TIMEOUT, ps_out, log_out + ), + ) + + +def image_exists(duthost, image_ref): + # Quote for refs that contain special chars + out = duthost.command( + "sudo docker image inspect {}".format(image_ref), module_ignore_errors=True + ) + return out["rc"] == 0 + + +def container_name_from_cfg(cfg): + val = cfg.get("validation") or {} + return val.get("docker_container_name", (cfg.get("docker_run") or {}).get("container_name")) + + +def _docker_exact_name_filter(container_name): + """Docker ``ps`` filter for an exact container name (not substring match).""" + return "name=^/{}$".format(container_name) + + +def list_live_addon_container_ids(duthost, container_name, running_only=False): + """ + Return container IDs whose name matches *container_name* exactly. + + Uses anchored ``name=^/$`` so only the configured container name matches (not substrings). + """ + if not container_name: + return [] + name_filter = _docker_exact_name_filter(container_name) + ps_cmd = "sudo docker ps" + if not running_only: + ps_cmd += " -a" + cmd = "{} --filter {} -q".format(ps_cmd, shlex.quote(name_filter)) + out = duthost.command(cmd, module_ignore_errors=True) + return [line.strip() for line in (out.get("stdout") or "").splitlines() if line.strip()] + + +def list_running_container_ids_by_image(duthost, image_ref): + """Return IDs of running containers started from *image_ref* (``ancestor`` filter).""" + if not image_ref: + return [] + image_filter = "ancestor={}".format(image_ref) + cmd = "sudo docker ps --filter {} -q".format(shlex.quote(image_filter)) + out = duthost.command(cmd, module_ignore_errors=True) + return [line.strip() for line in (out.get("stdout") or "").splitlines() if line.strip()] + + +def verify_single_live_addon_container_instance(duthost, cfg): + """ + Assert ``docker run`` / start did not leave multiple live-addon container instances. + + Checks exact container name and (by default) running containers from ``docker_run.image_ref``. + """ + cname = container_name_from_cfg(cfg) + if not cname: + logger.warning("verify_single_live_addon_container_instance: no container name in cfg; skipping") + return + + val = cfg.get("validation") or {} + expected = int(val.get("max_running_instances", 1)) + if expected < 0: + return + + image_ref = (cfg.get("docker_run") or {}).get("image_ref") + enforce_image = val.get("enforce_single_image_instance", True) + + running_by_name = list_live_addon_container_ids(duthost, cname, running_only=True) + all_by_name = list_live_addon_container_ids(duthost, cname, running_only=False) + running_by_image = list_running_container_ids_by_image(duthost, image_ref) if image_ref else [] + + logger.info( + "Live-addon instance check %r: running_by_name=%s all_by_name=%s running_by_image=%s (image=%r)", + cname, + running_by_name, + all_by_name, + running_by_image, + image_ref, + ) + + pytest_assert( + len(running_by_name) == expected, + "Expected exactly {} running container(s) named {!r}, found {} (ids={}). " + "Check for duplicate docker run/start.".format( + expected, cname, len(running_by_name), running_by_name + ), + ) + pytest_assert( + len(all_by_name) <= expected, + "Expected at most {} container record(s) named {!r}, found {} (ids={}). " + "Stale stopped containers or duplicate names may be present.".format( + expected, cname, len(all_by_name), all_by_name + ), + ) + if enforce_image and image_ref: + pytest_assert( + len(running_by_image) == expected, + "Expected exactly {} running container(s) from image {!r}, found {} (ids={}). " + "Multiple live-addon instances may have been started.".format( + expected, image_ref, len(running_by_image), running_by_image + ), + ) + + +def _normalize_supervisor_program_name(name): + """Normalize program names for comparison (``health_monitor`` vs ``health-monitor``).""" + return (name or "").replace("_", "-").lower() + + +def get_supervisorctl_status(duthost, container_name): + """ + Return ``(program_status_map, raw_output)`` from ``docker exec supervisorctl status``. + + *program_status_map* keys are supervisord program names; values are status strings (e.g. ``RUNNING``). + """ + cname = shlex.quote(container_name) + out = duthost.command( + "sudo docker exec {} supervisorctl status".format(cname), + module_ignore_errors=True, + ) + raw = _command_stdout_stderr(out) + statuses = {} + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split(None, 2) + if len(parts) < 2: + continue + program, status = parts[0], parts[1] + statuses[program] = status + return statuses, raw + + +def _find_supervisor_program_status(statuses, expected_program): + """Match *expected_program* to a supervisord entry (exact name, ignoring ``group:`` prefix).""" + norm_expected = _normalize_supervisor_program_name(expected_program) + for program, status in statuses.items(): + base = program.split(":")[-1] + if _normalize_supervisor_program_name(base) == norm_expected: + return program, status + return None, None + + +def _evaluate_live_addon_processes(statuses, expected_processes): + """ + Compare *expected_processes* to *statuses* from ``supervisorctl status``. + + Returns ``(missing, not_running)`` where *not_running* entries are + ``(expected_name, supervisord_program, status)``. + """ + expected = expected_processes or DEFAULT_LIVE_ADDON_EXPECTED_PROCESSES + missing = [] + not_running = [] + for proc in expected: + program, status = _find_supervisor_program_status(statuses, proc) + if program is None: + missing.append(proc) + elif status != "RUNNING": + not_running.append((proc, program, status)) + return missing, not_running + + +def _check_live_addon_container_processes(duthost, container_name, expected_processes=None): + """ + Query supervisord once; return ``(ok, statuses, raw, missing, not_running)``. + + *ok* is False when ``supervisorctl status`` is empty or expected programs are not ``RUNNING``. + """ + if expected_processes is not None and len(expected_processes) == 0: + return True, {}, "", [], [] + statuses, raw = get_supervisorctl_status(duthost, container_name) + if not statuses: + return False, {}, raw, [], [] + missing, not_running = _evaluate_live_addon_processes(statuses, expected_processes) + return (not missing and not not_running), statuses, raw, missing, not_running + + +def wait_for_live_addon_container_processes( + duthost, + container_name, + expected_processes=None, + timeout=DEFAULT_PROCESS_WAIT_SECONDS, + interval=DEFAULT_PROCESS_POLL_INTERVAL_SECONDS, +): + """Poll until expected supervisord programs are ``RUNNING``, then assert on timeout.""" + if expected_processes is not None and len(expected_processes) == 0: + return + + logger.info( + "Waiting up to %ss for supervisord programs RUNNING in container %r", + timeout, + container_name, + ) + last_raw = "" + + def _ready(): + nonlocal last_raw + ok, _statuses, raw, _missing, _not_running = _check_live_addon_container_processes( + duthost, container_name, expected_processes + ) + if ok: + last_raw = raw + return ok + + if not wait_until(timeout, interval, 0, _ready): + verify_live_addon_container_processes(duthost, container_name, expected_processes=expected_processes) + return + + logger.info("Live-addon container %r supervisord programs ready:\n%s", container_name, last_raw) + + +def verify_live_addon_container_processes(duthost, container_name, expected_processes=None): + """Assert expected supervisord programs are ``RUNNING`` (``docker exec … supervisorctl status``).""" + _ok, statuses, raw, missing, not_running = _check_live_addon_container_processes( + duthost, container_name, expected_processes + ) + if expected_processes is not None and len(expected_processes) == 0: + return + logger.info("Live-addon container %r supervisorctl status:\n%s", container_name, raw) + pytest_assert( + statuses, + "Container {!r} supervisorctl status is empty or unavailable".format(container_name), + ) + pytest_assert( + not missing, + "Container {!r} missing expected supervisord program(s) {} (supervisorctl: {!r})".format( + container_name, missing, statuses + ), + ) + pytest_assert( + not not_running, + "Container {!r} supervisord program(s) not RUNNING: {} (supervisorctl: {!r})".format( + container_name, not_running, statuses + ), + ) + + +def _command_stdout_stderr(result): + """Merge ansible command stdout and stderr (no shell redirect; ``command`` uses argv).""" + stdout = (result.get("stdout") or "").strip() + stderr = (result.get("stderr") or "").strip() + if stdout and stderr: + return stdout + "\n" + stderr + return stdout or stderr + + +def _is_valid_container_started_at(started_at): + """True when *started_at* looks like docker ``State.StartedAt`` RFC3339, not a broken template.""" + if not started_at: + return False + if started_at.startswith("0001-01-01"): + return False + if "{" in started_at or "}" in started_at: + return False + return bool(_STARTED_AT_TIMESTAMP_RE.match(started_at)) + + +def get_container_started_at(duthost, container_name): + """ + Return ``State.StartedAt`` (RFC3339) for the current container instance from ``docker inspect``. + + Used with ``docker logs --since`` so pattern checks ignore log lines from prior runs/restarts. + """ + cname = shlex.quote(container_name) + # Go template requires ``{{.State.StartedAt}}``; do not pass through str.format (collapses braces). + fmt = shlex.quote(_DOCKER_INSPECT_STARTED_AT_FMT) + out = duthost.command( + "sudo docker inspect -f {} {}".format(fmt, cname), + module_ignore_errors=True, + ) + if out.get("rc") != 0: + return None + started_at = (out.get("stdout") or "").strip() + if not _is_valid_container_started_at(started_at): + logger.warning( + "docker inspect StartedAt for %r is invalid (%r); will not use docker logs --since", + container_name, + started_at, + ) + return None + return started_at + + +def _logs_since_latest_session(logs, session_start_pattern): + """Keep log text from the last occurrence of *session_start_pattern* (fallback when ``--since`` unavailable).""" + if not session_start_pattern or not logs: + return logs + haystack = logs.lower() + needle = session_start_pattern.lower() + idx = haystack.rfind(needle) + if idx < 0: + return logs + return logs[idx:] + + +def fetch_container_logs(duthost, container_name, tail=None, since=None): + """ + Return ``docker logs`` stdout for *container_name*. + + Prefer ``since`` (container ``StartedAt``) to scope logs to the current run; ``tail`` applies only + when ``since`` is not set. + """ + cname = shlex.quote(container_name) + parts = ["sudo", "docker", "logs"] + if since: + parts.extend(["--since", shlex.quote(since)]) + elif tail is not None: + parts.extend(["--tail", str(int(tail))]) + parts.append(cname) + cmd = " ".join(parts) + out = duthost.command(cmd, module_ignore_errors=True) + return _command_stdout_stderr(out) + + +def startup_log_validation_enabled(val): + """True when vendor JSON defines non-empty ``validation.startup_log.required_patterns``.""" + val = val or {} + slog = val.get("startup_log") + if not slog: + return False + required = slog.get("required_patterns") + return bool(required) + + +def resolve_startup_log_validation(startup_log_cfg): + """ + Build startup log check settings from vendor ``validation.startup_log``. + + ``required_patterns`` and ``forbidden_patterns`` are vendor-specific (JSON only). + ``wait_seconds`` is the max time to wait for **all** required patterns; vendors override when + startup depends on external readiness (e.g. syncd container uptime before online diagnostic). + """ + slog = startup_log_cfg or {} + return { + "required_patterns": list(slog.get("required_patterns") or []), + "forbidden_patterns": list(slog.get("forbidden_patterns") or []), + "wait_seconds": int(slog.get("wait_seconds", DEFAULT_STARTUP_LOG_WAIT_SECONDS)), + "poll_interval_seconds": float( + slog.get("poll_interval_seconds", DEFAULT_STARTUP_LOG_POLL_INTERVAL_SECONDS) + ), + "log_tail": slog.get("log_tail"), + "session_start_pattern": slog.get("session_start_pattern"), + } + + +def _match_log_patterns(logs, patterns, case_insensitive=True): + """Return patterns from *patterns* that appear in *logs* (substring match).""" + haystack = logs.lower() if case_insensitive else logs + hits = [] + for pattern in patterns: + needle = pattern.lower() if case_insensitive else pattern + if needle in haystack: + hits.append(pattern) + return hits + + +def verify_live_addon_container_startup_logs(duthost, container_name, startup_log_cfg=None): + """ + Poll container logs every ``poll_interval_seconds`` until all ``required_patterns`` appear or + ``wait_seconds`` elapses (vendor JSON overrides timing; code default is generic only). + + Log lines are scoped to the current container run via ``docker logs --since ``. + """ + if not startup_log_cfg or not startup_log_cfg.get("required_patterns"): + logger.info("Startup log validation skipped (no vendor validation.startup_log.required_patterns)") + return + + cfg = resolve_startup_log_validation(startup_log_cfg) + required = cfg["required_patterns"] + forbidden = cfg["forbidden_patterns"] + wait_seconds = cfg["wait_seconds"] + poll_interval = cfg["poll_interval_seconds"] + log_tail = cfg["log_tail"] + session_start_pattern = cfg.get("session_start_pattern") + started_at = get_container_started_at(duthost, container_name) + if started_at: + logger.info( + "Startup log validation for %r: current run StartedAt=%s (docker logs --since)", + container_name, + started_at, + ) + elif session_start_pattern: + logger.warning( + "Startup log validation for %r: StartedAt unavailable; slicing from last %r", + container_name, + session_start_pattern, + ) + else: + logger.warning( + "Startup log validation for %r: StartedAt unavailable and no session_start_pattern; " + "checking full docker logs (may include prior runs)", + container_name, + ) + logger.info( + "Startup log validation for %r: wait up to %ss (poll every %ss) for %s required pattern(s)", + container_name, + wait_seconds, + poll_interval, + len(required), + ) + + if not required and not forbidden: + logger.info("Startup log validation disabled (empty required_patterns)") + return + + def _fetch_current_logs(): + if started_at: + return fetch_container_logs(duthost, container_name, since=started_at) + logs = fetch_container_logs(duthost, container_name, tail=log_tail) + if session_start_pattern: + return _logs_since_latest_session(logs, session_start_pattern) + return logs + + last_logs = "" + last_missing = list(required) + + def _startup_logs_ready(): + nonlocal last_logs, last_missing + last_logs = _fetch_current_logs() + forbidden_hits = _match_log_patterns(last_logs, forbidden) + if forbidden_hits: + snippet = last_logs[-_STARTUP_LOG_FAILURE_SNIPPET_CHARS:] + logger.error( + "Live-addon container %r startup logs — forbidden pattern(s) %s:\n%s", + container_name, + forbidden_hits, + snippet, + ) + pytest_assert( + False, + "Container {!r} logs contain failure pattern(s) {}. See test log for docker logs snippet.".format( + container_name, forbidden_hits + ), + ) + + matched = _match_log_patterns(last_logs, required) + last_missing = [p for p in required if p not in matched] + if not last_missing: + return True + + logger.debug( + "Live-addon container %r waiting for log patterns (missing %s); retry in %ss", + container_name, + last_missing, + poll_interval, + ) + return False + + if wait_until(wait_seconds, poll_interval, 0, _startup_logs_ready): + logger.info( + "Live-addon container %r startup logs OK (required patterns matched within %ss)", + container_name, + wait_seconds, + ) + logger.info( + "Live-addon container %r startup logs (last %s chars):\n%s", + container_name, + min(len(last_logs), 4000), + last_logs[-4000:], + ) + return + + snippet = last_logs[-_STARTUP_LOG_FAILURE_SNIPPET_CHARS:] + logger.error( + "Live-addon container %r startup logs — still missing %s after %ss:\n%s", + container_name, + last_missing, + wait_seconds, + snippet, + ) + pytest_assert( + False, + ( + "Container {!r} logs missing required pattern(s) {} after {}s wait. " + "See test log for docker logs snippet." + ).format(container_name, last_missing, wait_seconds), + ) + + +def verify_live_addon_post_start(duthost, cfg, full_readiness=True): + """ + Run post-``docker run`` checks: single instance, then readiness from JSON. + + Called after every ``docker run`` (fixture, config-reload cycle, restarts). + + When ``validation.startup_log`` is enabled and *full_readiness=True*, poll log patterns first, + then assert ``expected_processes`` via ``supervisorctl`` (one-shot after logs pass). + + When startup logs are skipped (*full_readiness=False* on restart, or no log config), poll + ``expected_processes`` for up to ``DEFAULT_PROCESS_WAIT_SECONDS`` (120s). + """ + cname = container_name_from_cfg(cfg) + if not cname: + logger.warning("verify_live_addon_post_start: no container name in cfg; skipping") + return + + verify_single_live_addon_container_instance(duthost, cfg) + + val = cfg.get("validation") or {} + expected_procs = expected_processes_from_validation(val) + startup_log_cfg = val.get("startup_log") + logs_enabled = full_readiness and startup_log_validation_enabled(val) + + if logs_enabled: + verify_live_addon_container_startup_logs(duthost, cname, startup_log_cfg=startup_log_cfg) + + if not expected_procs: + return + + if logs_enabled: + verify_live_addon_container_processes(duthost, cname, expected_processes=expected_procs) + else: + wait_for_live_addon_container_processes(duthost, cname, expected_processes=expected_procs) + + +def build_health_check_curl_command(health_cfg): + """curl used for validation; fields from JSON ``health`` section.""" + port = int(health_cfg["port"]) + path = health_cfg.get("url_path", "/health") + host = health_cfg.get("bind_host", "127.0.0.1") + return ( + "curl -sS -m 15 -o /tmp/live_addon_docker_health.out -w '%{{http_code}}' " + "http://{}:{}{}".format(host, port, path) + ) + + +def http_health_check(duthost, health_cfg): + """ + Query HTTP health endpoint from the DUT (container typically uses --net=host). + + Returns (ok: bool, http_code: str, body_snippet: str). Uses ``module_ignore_errors`` so + connection failures during polling do not raise (curl rc 7 → code ``000``, ok False). + """ + expect = str(health_cfg.get("expect_http_code", 200)) + curl = build_health_check_curl_command(health_cfg) + out = duthost.command(curl, module_ignore_errors=True) + code = (out.get("stdout") or "").strip() + if out.get("rc", 0) != 0 and not code: + code = "000" + body = duthost.command("sudo cat /tmp/live_addon_docker_health.out", module_ignore_errors=True).get( + "stdout", "" + )[:500] + ok = code == expect + return ok, code, body + + +def run_config_reload_live_addon_start_reload_health(duthost, resolved_cfg, loganalyzer=None): + """ + Config-reload cycle for live-addon persistence: + + 1. Stop/remove live-addon container (``docker_run.container_name``; fixture may still have it running). + 2. First ``config reload`` (``safe_reload`` waits for critical services). + 3. ``docker run`` live-addon + post-start checks (logs/processes). + 4. Second ``config reload``. + 5. ``docker_manual_teardown`` + ``docker_run_manual`` + post-start (process poll only). + 6. HTTP health probe (``wait_for_health_ready``). + """ + from tests.common.config_reload import config_reload + + docker_manual_teardown(duthost, resolved_cfg["docker_run"]) + logger.info("First config reload (live-addon container stopped)") + config_reload( + duthost, + config_source="config_db", + safe_reload=True, + ignore_loganalyzer=loganalyzer, + ) + + cfg_run = copy.deepcopy(resolved_cfg) + docker_run_manual(duthost, cfg_run) + verify_live_addon_post_start(duthost, cfg_run) + logger.info("Second config reload (live-addon container was running)") + config_reload( + duthost, + config_source="config_db", + safe_reload=True, + ignore_loganalyzer=loganalyzer, + ) + logger.info("Re-start live-addon container after second config reload (teardown + docker run)") + docker_manual_teardown(duthost, cfg_run["docker_run"]) + docker_run_manual(duthost, cfg_run) + verify_live_addon_post_start(duthost, cfg_run, full_readiness=False) + health_cfg = resolved_cfg.get("health") or {} + return wait_for_health_ready(duthost, health_cfg) + + +def wait_for_health_ready(duthost, health_cfg): + """ + Honor ``health.wait_seconds_before_check``, then HTTP probe(s) per + ``probe_timeout_seconds`` / ``probe_interval_seconds``. + Returns (ok, http_code, body) from the last attempt. + + Call ``verify_live_addon_post_start`` before this when the container was just started. + """ + initial = int(health_cfg.get("wait_seconds_before_check", 0)) + if initial > 0: + logger.info("Health: sleeping %s s before first probe (JSON wait_seconds_before_check)", initial) + time.sleep(initial) + + timeout = int(health_cfg.get("probe_timeout_seconds", 0)) + interval = int(health_cfg.get("probe_interval_seconds", 30)) + if timeout <= 0: + return http_health_check(duthost, health_cfg) + + last = (False, "", "") + + def _probe(): + nonlocal last + last = http_health_check(duthost, health_cfg) + return last[0] + + polled = wait_until(timeout, interval, 0, _probe) + pytest_assert(polled, "Health endpoint did not return expect_http_code within {} s".format(timeout)) + return last + + +def docker_manual_teardown(duthost, docker_run_cfg): + name = docker_run_cfg["container_name"] + logger.info("Teardown docker: stop and remove %s", name) + duthost.command("sudo docker stop {} 2>/dev/null || true".format(name), module_ignore_errors=True) + duthost.command("sudo docker rm -f {}".format(name), module_ignore_errors=True) + + +def remove_configured_live_addon_images(duthost, cfg): + """ + Best-effort ``docker rmi -f`` for ``docker_run.image_ref`` and ``candidate_image_refs``. + Used immediately before ``docker load`` so the tarball load does not layer on old tags. + """ + for ref in _image_refs_to_try(cfg): + logger.info("Removing live-addon image before docker load (best-effort): %s", ref) + duthost.command("sudo docker rmi -f {}".format(ref), module_ignore_errors=True) + + +def prepare_live_addon_docker_install(duthost, cfg, install_source): + """ + At test start: stop and remove the live-addon container if it is still running. + When install uses a tarball (``docker load``), remove configured image refs before load. + When using an image already on the DUT (no tarball), only the container is removed. + """ + dr = cfg.get("docker_run") + if not dr: + return + docker_manual_teardown(duthost, dr) + if install_source.kind == "image_present": + return + remove_configured_live_addon_images(duthost, cfg) + + +def get_core_filenames(duthost): + """Filenames under /var/core/ (same rules as platform tests).""" + if "20191130" in duthost.os_version: + out = duthost.shell("ls /var/core/ 2>/dev/null | grep -v python || true")["stdout"] + else: + out = duthost.shell("ls /var/core/ 2>/dev/null || true")["stdout"] + return set(line.strip() for line in out.splitlines() if line.strip()) + + +def verify_no_new_core_files(duthost, pre_cores): + post = get_core_filenames(duthost) + new_files = post - pre_cores + pytest_assert( + not new_files, + "New core file(s) appeared under /var/core/: {}".format(", ".join(sorted(new_files))), + ) + + +def verify_container_absent(duthost, container_name): + """Use ``SonicHost.get_all_containers()`` (escaped docker format, same as rest of sonic-mgmt).""" + if not container_name: + return + all_names = duthost.get_all_containers() + pytest_assert( + container_name not in all_names, + "Container {} still present after teardown".format(container_name), + ) + + +def verify_syslog_clean_after_teardown(duthost, cfg): + """ + Tail syslog; if any line mentions the live-addon (hints) and matches the default error pattern, fail. + """ + tail_lines = _DEFAULT_SYSLOG_TAIL_LINES + err_pat = _DEFAULT_SYSLOG_ERROR_PATTERN + hints = [] + val = cfg.get("validation") or {} + if val.get("docker_container_name"): + hints.append(val["docker_container_name"]) + dr = cfg.get("docker_run") or {} + if dr.get("container_name"): + cname = dr["container_name"] + if cname not in hints: + hints.append(cname) + image_ref = dr.get("image_ref", "") + if image_ref: + base = image_ref.split(":")[0].split("/")[-1] + if base and base not in hints: + hints.append(base) + if not hints: + return + hint_alt = "|".join(re.escape(h) for h in hints) + script = ( + "sudo tail -n {n} /var/log/syslog 2>/dev/null | grep -iE '({hints})' | grep -iE '{err}' || true" + ).format(n=tail_lines, hints=hint_alt, err=err_pat) + bad = duthost.command(script, module_ignore_errors=True).get("stdout", "").strip() + pytest_assert( + not bad, + "Syslog lines after teardown matched error pattern for live-addon hints:\n{}".format(bad[:2000]), + ) + + +def verify_post_teardown(duthost, cfg, pre_core_filenames): + """ + After stop/uninstall: assert no new core files, live-addon container absent, syslog spot-check. + Behavior is fixed in code (not vendor JSON). + + Relation to sonic-mgmt infra (keep these checks): + - Loganalyzer runs per test and does not analyze syslog after the last test returns; uninstall + in the module fixture runs after that, so a tail+grep for live-addon hints still adds coverage. + - core_dump_and_config_check (module autouse) compares /var/core at module boundaries but does + not pytest.fail on new cores; it logs and records cache for telemetry. The assert here fails + the run if new cores appeared during this fixture. + """ + val = cfg.get("validation") or {} + cname = val.get("docker_container_name", (cfg.get("docker_run") or {}).get("container_name")) + + verify_no_new_core_files(duthost, pre_core_filenames) + verify_container_absent(duthost, cname) + verify_syslog_clean_after_teardown(duthost, cfg) diff --git a/tests/live_addon_docker/test_live_addon_docker.py b/tests/live_addon_docker/test_live_addon_docker.py new file mode 100644 index 00000000000..8d56cdf0ba2 --- /dev/null +++ b/tests/live_addon_docker/test_live_addon_docker.py @@ -0,0 +1,87 @@ +""" +Install and validate a live-addon docker via ``docker run`` (no sonic-package-manager). + +Image resolution (registry first by default, then fallbacks): + +1. **Registry** — ``docker pull`` from ``--live_addon_docker_registry`` or Ansible ``docker_registry_*`` +2. **Tarball on DUT** — ``/home/admin/`` +3. **Tarball on test runner** — copy to ``/tmp/`` on DUT +4. **Image already on DUT** — ``docker image inspect`` + +Pass ``--live_addon_docker_image_tag`` for CI build tags; ``--live_addon_docker_registry`` per-vendor CR. + +Post-start validation (single instance, startup logs or supervisord poll) runs in the module fixture +and on every ``docker run`` inside helpers — not duplicated in individual tests below. +""" + +import copy +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.live_addon_docker import live_addon_docker_helpers as lad + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology("any"), +] + + +@pytest.fixture(scope="module") +def live_addon_docker_setup_teardown(live_addon_docker_install_source, live_addon_docker_vendor_cfg): + duthost, src = live_addon_docker_install_source + cfg0 = live_addon_docker_vendor_cfg + pre_cores = lad.get_core_filenames(duthost) + cfg = None + + try: + lad.prepare_live_addon_docker_install(duthost, cfg0, src) + if src.kind == "image_present": + ref = src.image_ref + cfg = copy.deepcopy(cfg0) + cfg["docker_run"]["image_ref"] = ref + else: + remote_tarball = src.remote_tarball_path + load_out = lad.docker_load(duthost, remote_tarball) + ref = lad.parse_image_from_docker_load(load_out) + cfg = copy.deepcopy(cfg0) + if ref: + cfg["docker_run"]["image_ref"] = ref + + lad.require_version_matrix_or_skip(duthost, cfg0, cfg["docker_run"]["image_ref"]) + lad.docker_run_manual(duthost, cfg) + lad.verify_live_addon_post_start(duthost, cfg) + + yield duthost, cfg + + finally: + try: + if cfg is not None: + lad.docker_manual_teardown(duthost, cfg["docker_run"]) + except Exception as exc: + logger.warning("Teardown command failed (cleanup checks still run): %s", exc) + lad.verify_post_teardown(duthost, cfg if cfg is not None else cfg0, pre_cores) + + +def test_live_addon_docker_health_http(live_addon_docker_setup_teardown): + duthost, cfg = live_addon_docker_setup_teardown + ok, code, body = lad.wait_for_health_ready(duthost, cfg["health"]) + pytest_assert(ok, "Health check failed: http_code={} body={}".format(code, body)) + + +def test_live_addon_docker_health_after_config_reload_cycle( + live_addon_docker_setup_teardown, loganalyzer +): + """ + Config reload, start live-addon via ``docker run``, config reload again, + teardown + ``docker run`` again, then validate HTTP health. + """ + duthost, cfg = live_addon_docker_setup_teardown + ok, code, body = lad.run_config_reload_live_addon_start_reload_health( + duthost, cfg, loganalyzer=loganalyzer + ) + pytest_assert( + ok, + "Health check after config-reload cycle failed: http_code={} body={}".format(code, body), + ) diff --git a/tests/lldp/test_lldp_syncd.py b/tests/lldp/test_lldp_syncd.py index 5d6e930b1be..dc4ad63bb2c 100644 --- a/tests/lldp/test_lldp_syncd.py +++ b/tests/lldp/test_lldp_syncd.py @@ -2,6 +2,7 @@ # Test plan in docs/testplan/LLDP-syncd-test-plan.md import pytest import json +import time from tests.common.helpers.sonic_db import SonicDbCli import logging from tests.common.reboot import reboot, REBOOT_TYPE_COLD @@ -11,7 +12,6 @@ group_interfaces_by_asic ) from tests.common.helpers.assertions import pytest_assert -import time logger = logging.getLogger(__name__) @@ -425,7 +425,12 @@ def test_lldp_entry_table_content( # Test case 2: Verify LLDP_ENTRY_TABLE after restart syncd and orchagent +# This test deliberately restarts swss/syncd, which cascades to a bgp container +# restart. The memory_utilization fixture's before/after snapshots become +# meaningless across such a restart (bgpd RSS drops to ~0 then warms back up), +# so disable memory monitoring for this test. @pytest.mark.disable_loganalyzer +@pytest.mark.disable_memory_utilization def test_lldp_entry_table_after_syncd_orchagent( duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance ): @@ -451,7 +456,14 @@ def test_lldp_entry_table_after_syncd_orchagent( duthost.shell("sudo systemctl restart swss") assert wait_until(600, 5, 120, duthost.critical_services_fully_started), \ "Not all critical services are fully started" - time.sleep(60) + # Wait for BGP sessions to reach Established state instead of a fixed sleep, + # to avoid a downstream memory-alarm false positive caused by bgpd warming up + # in the next test case. + bgp_neighbors = list(duthost.get_bgp_neighbors().keys()) + pytest_assert( + wait_until(300, 10, 30, duthost.check_bgp_session_state, bgp_neighbors), + "BGP sessions did not reach Established state after swss restart", + ) # Wait until all interfaces are up and lldp entries are populated for interface in lldp_entry_keys: result = wait_until(300, 2, 0, verify_lldp_entry, db_instance, [interface]) @@ -525,7 +537,7 @@ def test_lldp_entry_table_after_all_batched_flap( # Test case 5: Verify LLDP_ENTRY_TABLE after system reboot def test_lldp_entry_table_after_lldp_restart( - duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance + duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance, ignore_expected_loganalyzer_exceptions, ): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] lldp_entry_keys, show_lldp_table_int_list, lldpctl_output = get_lldp_data(duthost, db_instance) diff --git a/tests/macsec/conftest.py b/tests/macsec/conftest.py index d7b1bc6b2a6..7e93ffd5169 100644 --- a/tests/macsec/conftest.py +++ b/tests/macsec/conftest.py @@ -22,6 +22,23 @@ def profile_name(macsec_profile): return macsec_profile['name'] +@pytest.fixture(scope="module") +def get_port_profile_name(macsec_profile, port_profiles): + """Return a callable ``f(dut_port)`` that resolves the MACsec profile + name for a given port. In single-profile mode this always returns the + same name. Tests that disable/re-enable MACsec on a port should use + this instead of ``profile_name``. + """ + if port_profiles: + def _resolve(dut_port): + return port_profiles[dut_port]['name'] + else: + name = macsec_profile['name'] + def _resolve(dut_port): # noqa: E306 + return name + return _resolve + + @pytest.fixture(scope="module") def default_priority(macsec_profile): return macsec_profile['priority'] @@ -58,5 +75,21 @@ def rekey_period(macsec_profile): @pytest.fixture(scope="module") -def wait_mka_establish(duthost, ctrl_links, policy, cipher_suite, send_sci): - assert wait_until(300, 6, 12, check_appl_db, duthost, ctrl_links, policy, cipher_suite, send_sci) +def wait_mka_establish(duthost, ctrl_links, port_profiles, policy, + cipher_suite, send_sci): + if port_profiles: + # If per interface, verify that each port is bound to its + # per-interface profile in CONFIG_DB. + from tests.common.macsec.macsec_helper import getns_prefix + for dut_port, profile in port_profiles.items(): + cmd = "sonic-db-cli {} CONFIG_DB HGET 'PORT|{}' 'macsec'".format( + getns_prefix(duthost, dut_port), dut_port) + bound_profile = duthost.command(cmd)['stdout'].strip() + assert bound_profile == profile['name'], \ + "Port {} bound to '{}', expected '{}'".format( + dut_port, bound_profile, profile['name']) + + # Validate APPL_DB tables — works for both single-profile and + # per-interface mode since cipher_suite/policy/send_sci are uniform. + assert wait_until(300, 6, 12, check_appl_db, duthost, ctrl_links, + policy, cipher_suite, send_sci) diff --git a/tests/macsec/test_controlplane.py b/tests/macsec/test_controlplane.py index f1fa635196b..ff441692e9c 100644 --- a/tests/macsec/test_controlplane.py +++ b/tests/macsec/test_controlplane.py @@ -89,9 +89,11 @@ def test_rekey_by_period(self, duthost, ctrl_links, upstream_links, rekey_period duthost.command("rm {}".format(tmp_file)) @pytest.mark.disable_loganalyzer - def test_profile_replace(self, duthost, ctrl_links, + def test_profile_replace(self, duthost, ctrl_links, port_profiles, profile_name, default_priority, cipher_suite, primary_cak, primary_ckn, policy, send_sci, rekey_period, tbinfo, wait_mka_establish): + if port_profiles: + pytest.skip("Per-interface profile replacement tested in test_per_interface_profile") # Only pick one controlled link for profile replace test ctrl_link = dict([next(iter(ctrl_links.items()))]) port_name, nbr = list(ctrl_link.items())[0] diff --git a/tests/macsec/test_fault_handling.py b/tests/macsec/test_fault_handling.py index 1ace979d1b7..f15f64dfafd 100644 --- a/tests/macsec/test_fault_handling.py +++ b/tests/macsec/test_fault_handling.py @@ -138,9 +138,11 @@ def check_new_mka_session(): ) @pytest.mark.disable_loganalyzer - def test_mismatch_macsec_configuration(self, duthost, unctrl_links, + def test_mismatch_macsec_configuration(self, duthost, unctrl_links, port_profiles, profile_name, default_priority, cipher_suite, primary_cak, primary_ckn, policy, send_sci, wait_mka_establish): + if port_profiles: + pytest.skip("Mismatch test uses single-profile CAK/CKN fixtures") # Only pick one uncontrolled link for mismatch macsec configuration test if not unctrl_links: pytest.skip('SKIP this test as there are no uncontrolled links in this dut') diff --git a/tests/macsec/test_interop_protocol.py b/tests/macsec/test_interop_protocol.py index 33621520e84..fa529517a05 100644 --- a/tests/macsec/test_interop_protocol.py +++ b/tests/macsec/test_interop_protocol.py @@ -23,7 +23,7 @@ class TestInteropProtocol(): ''' @pytest.mark.disable_loganalyzer - def test_port_channel(self, duthost, profile_name, ctrl_links, wait_mka_establish): + def test_port_channel(self, duthost, get_port_profile_name, ctrl_links, wait_mka_establish): '''Verify lacp ''' ctrl_port, _ = list(ctrl_links.items())[0] @@ -44,7 +44,7 @@ def test_port_channel(self, duthost, profile_name, ctrl_links, wait_mka_establis ) ) - enable_macsec_port(duthost, ctrl_port, profile_name) + enable_macsec_port(duthost, ctrl_port, get_port_profile_name(ctrl_port)) # Add ethernet interface back to PortChannel interface duthost.command("sudo config portchannel {} member add {} {}" .format(getns_prefix(duthost, ctrl_port), pc["name"], ctrl_port)) @@ -59,7 +59,7 @@ def test_port_channel(self, duthost, profile_name, ctrl_links, wait_mka_establis ) @pytest.mark.disable_loganalyzer - def test_lldp(self, duthost, ctrl_links, profile_name, wait_mka_establish): + def test_lldp(self, duthost, ctrl_links, get_port_profile_name, wait_mka_establish): '''Verify lldp ''' LLDP_ADVERTISEMENT_INTERVAL = 30 # default interval in seconds @@ -100,8 +100,8 @@ def test_lldp(self, duthost, ctrl_links, profile_name, wait_mka_establish): nbr["name"], get_lldp_list(duthost) ) - enable_macsec_port(duthost, ctrl_port, profile_name) - enable_macsec_port(nbr["host"], nbr["port"], profile_name) + enable_macsec_port(duthost, ctrl_port, get_port_profile_name(ctrl_port)) + enable_macsec_port(nbr["host"], nbr["port"], get_port_profile_name(ctrl_port)) wait_until(20, 3, 0, lambda: duthost.iface_macsec_ok(ctrl_port) and nbr["host"].iface_macsec_ok(nbr["port"])) @@ -117,7 +117,7 @@ def test_lldp(self, duthost, ctrl_links, profile_name, wait_mka_establish): ) @pytest.mark.disable_loganalyzer - def test_bgp(self, duthost, ctrl_links, upstream_links, profile_name, wait_mka_establish): + def test_bgp(self, duthost, ctrl_links, upstream_links, get_port_profile_name, wait_mka_establish): '''Verify BGP neighbourship ''' bgp_config = list(duthost.get_running_config_facts()[ @@ -171,8 +171,8 @@ def check_bgp_established(ctrl_port, up_link): # Check the BGP sessions are present after port macsec enabled for ctrl_port, nbr in list(ctrl_links.items()): - enable_macsec_port(duthost, ctrl_port, profile_name) - enable_macsec_port(nbr["host"], nbr["port"], profile_name) + enable_macsec_port(duthost, ctrl_port, get_port_profile_name(ctrl_port)) + enable_macsec_port(nbr["host"], nbr["port"], get_port_profile_name(ctrl_port)) wait_until(BGP_TIMEOUT, 3, 0, lambda: duthost.iface_macsec_ok(ctrl_port) and nbr["host"].iface_macsec_ok(nbr["port"])) diff --git a/tests/macsec/test_per_interface_profile.py b/tests/macsec/test_per_interface_profile.py new file mode 100644 index 00000000000..fbc5848fa0f --- /dev/null +++ b/tests/macsec/test_per_interface_profile.py @@ -0,0 +1,181 @@ +import pytest +import logging +import time +import random + +from tests.common.utilities import wait_until +from tests.common.macsec.macsec_helper import get_appl_db, get_ipnetns_prefix, load_all_macsec_info, check_appl_db +from tests.common.macsec.macsec_config_helper import ( + generate_macsec_profile, + setup_macsec_multi_profile_configuration, + disable_macsec_port, + enable_macsec_port, + delete_macsec_profile, + replace_macsec_port +) + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.macsec_required, + pytest.mark.topology("t0", "t2", "t0-sonic"), +] + + +class TestPerInterfaceProfile(): + ''' + Tests specific to per-interface MACsec profile functionality. + Validates behaviours that only exist when different ports use + different profiles simultaneously. + Requires ``--per_interface_macsec`` to be set. + ''' + + def test_unique_keys(self, port_profiles, wait_mka_establish): + '''Verify that each port was configured with a unique CAK/CKN pair + ''' + if not port_profiles: + pytest.skip("Requires --per_interface_macsec") + profile_list = list(port_profiles.values()) + for i in range(len(profile_list)): + for j in range(i + 1, len(profile_list)): + assert profile_list[i]["primary_cak"] != profile_list[j]["primary_cak"], \ + "CAK collision between {} and {}".format( + profile_list[i]["name"], profile_list[j]["name"]) + assert profile_list[i]["primary_ckn"] != profile_list[j]["primary_ckn"], \ + "CKN collision between {} and {}".format( + profile_list[i]["name"], profile_list[j]["name"]) + + @pytest.mark.disable_loganalyzer + def test_profile_isolation(self, duthost, ctrl_links, upstream_links, + port_profiles, policy, cipher_suite, send_sci, tbinfo, wait_mka_establish): + '''Disable MACsec on one port and verify other ports remain operational + ''' + if not port_profiles: + pytest.skip("Requires --per_interface_macsec") + ports = list(ctrl_links.keys()) + target_port = random.choice(ports) + ports.remove(target_port) + surviving_port = random.choice(ports) + target_nbr = ctrl_links[target_port] + target_profile = port_profiles[target_port] + + if surviving_port in upstream_links: + up = upstream_links[surviving_port] + ret = duthost.command( + "{} ping -c 4 {}".format( + get_ipnetns_prefix(duthost, surviving_port), + up["local_ipv4_addr"]), module_ignore_errors=True) + assert not ret["failed"], \ + "Pre disable ping failed on surviving port {}".format(surviving_port) + + disable_macsec_port(duthost, target_port) + disable_macsec_port(target_nbr["host"], target_nbr["port"]) + + # Allow some time for port channels to come back up + time.sleep(30) + + try: + def _surviving_ok(): + nbr = ctrl_links[surviving_port] + pt, esc, isc, esa, isa = get_appl_db( + duthost, surviving_port, nbr["host"], nbr["port"]) + return bool(pt and esc and isc and pt.get("enable") == "true") + + assert wait_until(60, 3, 5, _surviving_ok), \ + "Surviving port {} lost MACsec after disabling {}".format( + surviving_port, target_port) + + if surviving_port in upstream_links: + up = upstream_links[surviving_port] + ret = duthost.command( + "{} ping -c 4 {}".format( + get_ipnetns_prefix(duthost, surviving_port), + up["local_ipv4_addr"]), module_ignore_errors=True) + assert not ret["failed"], \ + "Ping failed on surviving port {}".format(surviving_port) + finally: + enable_macsec_port(duthost, target_port, target_profile["name"]) + enable_macsec_port(target_nbr["host"], target_nbr["port"], + target_profile["name"]) + + assert wait_until(300, 3, 0, + lambda: duthost.iface_macsec_ok(target_port) and + target_nbr["host"].iface_macsec_ok(target_nbr["port"])), \ + "MACsec did not recover on {}".format(target_port) + + target_ctrl_links = {target_port: ctrl_links.get(target_port), + surviving_port: ctrl_links.get(surviving_port)} + assert wait_until(300, 3, 0, + check_appl_db, duthost, target_ctrl_links, policy, cipher_suite, send_sci),\ + "appl_db didnt recover" + + load_all_macsec_info(duthost, ctrl_links, tbinfo) + + @pytest.mark.disable_loganalyzer + def test_profile_replace(self, duthost, ctrl_links, port_profiles, + cipher_suite, policy, send_sci, default_priority, + rekey_period, tbinfo, wait_mka_establish): + '''Replace the profile on one port and verify other ports are unaffected + ''' + if not port_profiles: + pytest.skip("Requires --per_interface_macsec") + ports = list(ctrl_links.keys()) + target_port = random.choice(ports) + ports.remove(target_port) + other_port = random.choice(ports) + target_nbr = ctrl_links[target_port] + other_nbr = ctrl_links[other_port] + + _, _, _, orig_target_esa, _ = get_appl_db( + duthost, target_port, target_nbr["host"], target_nbr["port"]) + + new_profile = generate_macsec_profile( + port_name=target_port, + cipher_suite=cipher_suite, + priority=default_priority, + policy=policy, + send_sci=send_sci, + rekey_period=rekey_period, + ) + new_profile["name"] = "MACSEC_PROFILE_{}_NEW".format(target_port) + + new_port_profiles = {target_port: new_profile} + setup_macsec_multi_profile_configuration( + duthost, {target_port: target_nbr}, new_port_profiles, tbinfo) + + try: + def _check_new_sa(): + _, _, _, new_esa, _ = get_appl_db( + duthost, target_port, target_nbr["host"], + target_nbr["port"]) + if not new_esa: + return False + return new_esa != orig_target_esa + assert wait_until(60, 5, 2, _check_new_sa), \ + "SA keys did not change on {} after profile replace".format( + target_port) + + pt, _, _, other_esa, _ = get_appl_db( + duthost, other_port, other_nbr["host"], other_nbr["port"]) + assert other_esa, "Other port {} lost SA tables".format(other_port) + assert pt["cipher_suite"] == port_profiles[other_port]["cipher_suite"], \ + "Other port cipher changed unexpectedly" + finally: + # Replace profile on port back to the original, and clean up the test profile + orig_port_profile = port_profiles[target_port] + replace_macsec_port(duthost, target_port, orig_port_profile["name"]) + replace_macsec_port(target_nbr["host"], target_nbr["port"], orig_port_profile["name"]) + delete_macsec_profile(duthost, new_profile["name"]) + delete_macsec_profile(target_nbr["host"], new_profile["name"]) + + assert wait_until(300, 3, 0, + lambda: duthost.iface_macsec_ok(target_port) and + target_nbr["host"].iface_macsec_ok(target_nbr["port"])), \ + "MACsec did not recover on {}".format(target_port) + + assert wait_until(300, 3, 0, + check_appl_db, duthost, + {target_port: ctrl_links.get(target_port)}, policy, cipher_suite, send_sci), \ + "appl_db didnt recover" + + load_all_macsec_info(duthost, ctrl_links, tbinfo) diff --git a/tests/memory_checker/test_memory_checker.py b/tests/memory_checker/test_memory_checker.py index f759d91bffe..99d3f1f9abd 100644 --- a/tests/memory_checker/test_memory_checker.py +++ b/tests/memory_checker/test_memory_checker.py @@ -665,7 +665,22 @@ def test_memory_checker_recover(memory_checker_dut_and_container, test_setup_and loganalyzer.expect_regex = container.get_restart_expected_logre() marker = loganalyzer.init() - timeout_status_change = 30 # monit has a 5s cycle interval per test parameters + # monit has a 5s cycle interval per test parameters, so 30s (~6 cycles) + # is normally plenty for the status to change. The 4GB Marvell-Prestera + # Nokia-7215 / Nokia-M0-7215, however, run under higher memory/CPU + # pressure on newer branches (notably 202511) and monit cannot hold its + # 5s cadence during the memory-stress window, intermittently missing the + # 30s budget (~45% nightly flake; confirmed expected platform behavior + # with the Nokia-7215 platform owner). Give only those two SKUs extra + # headroom. The 8GB Nokia-7215-A1 variants are not affected and are + # intentionally excluded, matching the exact-SKU gating already used for + # these platforms in platform_tests/test_reboot.py and test_reload_config.py. + # wait_monit_mem_* early-exit on success, so the larger ceiling never adds + # runtime on a healthy run. + if duthost.facts["hwsku"] in {"Nokia-7215", "Nokia-M0-7215"}: + timeout_status_change = 200 + else: + timeout_status_change = 30 container.start_consume_memory() container.wait_monit_mem_last_failed(timeout_status_change) diff --git a/tests/monit/test_monit_status.py b/tests/monit/test_monit_status.py index df73d5c7dbd..c14ed8a87e5 100644 --- a/tests/monit/test_monit_status.py +++ b/tests/monit/test_monit_status.py @@ -127,5 +127,6 @@ def test_monit_reporting_message(duthosts, enum_rand_one_per_hwsku_frontend_host pytest_assert(wait_until(180, 60, 0, check_monit_last_output, duthost), "Expected Monit reporting message not found") - wait_until(180, 60, 0, check_monit_expected_container_logging, duthost) + pytest_assert(wait_until(180, 60, 0, check_monit_expected_container_logging, duthost), + "Monit logged unexpected container-not-running messages") logger.info("Checking the format of Monit alerting message was done!") diff --git a/tests/nat/conftest.py b/tests/nat/conftest.py index d35dd611542..6f41952e9cd 100644 --- a/tests/nat/conftest.py +++ b/tests/nat/conftest.py @@ -191,7 +191,7 @@ def apply_global_nat_config(duthost, config_nat_feature_enabled): nat_global_config(duthost) yield # reload config on teardown - config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True) + config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True, override_config=True) @pytest.fixture() @@ -207,7 +207,7 @@ def reload_dut_config(request, duthost, setup_test_env): dut_iface = setup_data[interface_type]["vrf_conf"]["red"]["dut_iface"] gw_ip = setup_data[interface_type]["vrf_conf"]["red"]["gw"] mask = setup_data[interface_type]["vrf_conf"]["red"]["mask"] - config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True) + config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True, override_config=True) pch_ip = setup_info["pch_ips"][dut_iface] duthost.shell("sudo config interface ip remove {} {}/31".format(dut_iface, pch_ip)) duthost.shell("sudo config interface ip add {} {}/{}".format(dut_iface, gw_ip, mask)) diff --git a/tests/ospf/conftest.py b/tests/ospf/conftest.py index c4a93c13212..bfe68b72b90 100644 --- a/tests/ospf/conftest.py +++ b/tests/ospf/conftest.py @@ -46,7 +46,7 @@ def trap_copp_ospf(duthosts, rand_one_dut_hostname): @pytest.fixture(scope="module") def ospf_Bfd_setup(duthosts, rand_one_dut_hostname, nbrhosts, trap_copp_ospf, request): - if request.config.getoption("neighbor_type") != "sonic": + if request.config.getoption("neighbor_type") not in ("sonic", "csonic"): pytest.skip("Neighbor type must be sonic") duthost = duthosts[rand_one_dut_hostname] @@ -129,7 +129,7 @@ def get_ospf_neighbor_interface(host): def ospf_setup(duthosts, rand_one_dut_hostname, nbrhosts, tbinfo, request): # verify neighbors are type sonic - if request.config.getoption("neighbor_type") != "sonic": + if request.config.getoption("neighbor_type") not in ("sonic", "csonic"): pytest.skip("Neighbor type must be sonic") duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/packet_trimming/constants.py b/tests/packet_trimming/constants.py index adfb7272fb5..38a30f8d214 100644 --- a/tests/packet_trimming/constants.py +++ b/tests/packet_trimming/constants.py @@ -27,7 +27,7 @@ DUMMY_MAC = "00:11:22:33:44:55" PACKET_COUNT = 1000 BATCH_PACKET_COUNT = 10000 -SEND_MAX_RETRIES = 3 +SEND_MAX_RETRIES = 10 ECN = 2 # ECN Capable Transport(0), ECT(0) PACKET_SIZE_MARGIN = 4 diff --git a/tests/packet_trimming/packet_trimming_helper.py b/tests/packet_trimming/packet_trimming_helper.py index 7cd19b1ba8c..57f264e4c0c 100644 --- a/tests/packet_trimming/packet_trimming_helper.py +++ b/tests/packet_trimming/packet_trimming_helper.py @@ -822,24 +822,35 @@ def fill_egress_buffer(duthost, ptfadapter, port_id, buffer_size, target_queue, # Try to send remaining packets if there are any and we haven't already given up if remaining_packets > 0 and batch_index >= num_batches: - try: - logger.info(f"Sending remaining {remaining_packets} packets") - for interface in interfaces: - fill_packet = interface_packets[interface] - testutils.send( - ptfadapter, - port_id=port_id, - pkt=fill_packet, - count=remaining_packets - ) - logger.info(f"Sent {remaining_packets} remaining packets for {interface}") - total_sent_packets += remaining_packets * len(interfaces) - except Exception as e: - logger.warning(f"Failed to send remaining packets: {e}") - # Not critical if we've already sent most packets + retries = 0 + remaining_success = False + while not remaining_success and retries < SEND_MAX_RETRIES: + try: + logger.info(f"Sending remaining {remaining_packets} packets") + for interface in interfaces: + fill_packet = interface_packets[interface] + testutils.send( + ptfadapter, + port_id=port_id, + pkt=fill_packet, + count=remaining_packets + ) + logger.info(f"Sent {remaining_packets} remaining packets for {interface}") + total_sent_packets += remaining_packets * len(interfaces) + remaining_success = True + except Exception as e: + retries += 1 + logger.warning(f"Remaining packets failed (attempt {retries}/{SEND_MAX_RETRIES}): {e}") + # Wait before retry + time.sleep(2) + ptfadapter.dataplane.flush() logger.info(f"Buffer filling completed, sent {total_sent_packets} packets") + # Fail fast when nothing was sent so the buffer never gets filled + if total_sent_packets == 0: + raise RuntimeError("Buffer fill failed: 0 packets sent") + # Check queue counters after filling for interface in interfaces: logger.info(f"Queue counters after filling for {interface}:") diff --git a/tests/passw_hardening/conftest.py b/tests/passw_hardening/conftest.py index f5027354e0d..8e665059c86 100644 --- a/tests/passw_hardening/conftest.py +++ b/tests/passw_hardening/conftest.py @@ -3,22 +3,20 @@ from . import passw_hardening_utils -def set_default_passw_hardening_policies(duthosts, enum_rand_one_per_hwsku_hostname): +@pytest.fixture(scope="module") +def passw_policies_snapshot(duthosts, enum_rand_one_per_hwsku_hostname, passw_version_required): + """Snapshot the DUT's current password hardening policies once per module. + + The teardown of clean_passw_policies restores exactly these values. Previously it + reset the policies to hard-coded "default" values; whenever those drifted from the + real SONiC boot defaults (defined in init_cfg.json.j2), the module-scoped config + check detected a CONFIG_DB diff and ran config_reload, which restarted BGP and + produced spurious "bgpd memory increased" alarms on the next, unrelated test. + Capturing the actual values keeps the restore correct even if the image defaults + change. + """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] - - passw_hardening_ob_dis = passw_hardening_utils.PasswHardening(state='disabled', - expiration='100', - expiration_warning='15', - history='12', - len_min='8', - reject_user_passw_match='true', - lower_class='true', - upper_class='true', - digit_class="true", - special_class='true') - - passw_hardening_utils.config_and_review_policies(duthost, passw_hardening_ob_dis, - passw_hardening_utils.PAM_PASSWORD_CONF_DEFAULT_EXPECTED) + return passw_hardening_utils.get_passw_policies(duthost) @pytest.fixture(scope="module", autouse=True) @@ -36,9 +34,10 @@ def passw_version_required(duthosts, enum_rand_one_per_hwsku_hostname): @pytest.fixture(scope="function") -def clean_passw_policies(duthosts, enum_rand_one_per_hwsku_hostname): +def clean_passw_policies(duthosts, enum_rand_one_per_hwsku_hostname, passw_policies_snapshot): yield - set_default_passw_hardening_policies(duthosts, enum_rand_one_per_hwsku_hostname) + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + passw_hardening_utils.restore_passw_policies(duthost, passw_policies_snapshot) @pytest.fixture(scope="function") diff --git a/tests/passw_hardening/passw_hardening_utils.py b/tests/passw_hardening/passw_hardening_utils.py index a58337fbeb8..7083dcd29a4 100755 --- a/tests/passw_hardening/passw_hardening_utils.py +++ b/tests/passw_hardening/passw_hardening_utils.py @@ -1,3 +1,4 @@ +import ast import logging import os import difflib @@ -50,6 +51,57 @@ def __init__(self, state='disabled', expiration='100', expiration_warning='15', } +def get_passw_policies(duthost): + """Snapshot the current PASSW_HARDENING|POLICIES hash from CONFIG_DB. + + Returns a dict keyed by CONFIG_DB field names, or None when the key is absent or + the output cannot be parsed (in which case the caller should skip restoration). + """ + result = duthost.shell('sonic-db-cli CONFIG_DB hgetall "PASSW_HARDENING|POLICIES"', + module_ignore_errors=True) + output = result['stdout'].strip() + if result['rc'] != 0 or not output: + logging.warning("Could not read PASSW_HARDENING|POLICIES from CONFIG_DB: %s", result.get('stderr')) + return None + try: + policies = ast.literal_eval(output) + except (ValueError, SyntaxError): + logging.warning("Could not parse PASSW_HARDENING|POLICIES output: %r", output) + return None + if not isinstance(policies, dict): + logging.warning("Unexpected PASSW_HARDENING|POLICIES output: %r", output) + return None + return policies + + +def restore_passw_policies(duthost, snapshot): + """Restore PASSW_HARDENING policies to the values captured by get_passw_policies(). + + Only fields whose live value differs from the snapshot are re-applied, so a test + that did not touch the policies issues zero CLI commands (and therefore creates no + CONFIG_DB diff that would trigger a config_reload on the next test). CONFIG_DB field + names use underscores while the `config passw-hardening policies` CLI uses hyphens, + so each field is converted with '_' -> '-'. 'state' is applied last so the feature + is only (re)enabled/disabled after every dependent field has been written. + """ + if not snapshot: + logging.warning("No password hardening policies snapshot to restore; skipping") + return + current = get_passw_policies(duthost) + # If the live state cannot be read, conservatively restore every snapshot field. + fields_to_restore = [field for field in snapshot + if current is None or snapshot[field] != current.get(field)] + if not fields_to_restore: + logging.info("Password hardening policies unchanged since snapshot; nothing to restore") + return + ordered_fields = [field for field in fields_to_restore if field != "state"] + if "state" in fields_to_restore: + ordered_fields.append("state") + for field in ordered_fields: + cli_key = field.replace("_", "-") + duthost.command("sudo config passw-hardening policies {} {}".format(cli_key, snapshot[field])) + + def config_user(duthost, username, mode='add'): """ Function add or rm users using useradd/userdel tool. """ diff --git a/tests/pc/test_lag_2.py b/tests/pc/test_lag_2.py index 1c000707cec..1075a1ffed0 100644 --- a/tests/pc/test_lag_2.py +++ b/tests/pc/test_lag_2.py @@ -85,6 +85,30 @@ def __check_shell_output(self, host, command): def __check_intf_state(self, vm_host, intf, expect): return vm_host.check_intf_link_state(vm_host, intf) == expect + def __resolve_vm_neighbor_intf(self, peer_device, neighbor_intf): + """Map minigraph neighbor port to the interface name on converged cEOS.""" + peer_info = self.nbrhosts.get(peer_device) or {} + if peer_info.get('is_multi_vrf_peer') and peer_info.get('multi_vrf_data'): + return peer_info['multi_vrf_data']['orig_intf_map'][neighbor_intf] + + props = self.tbinfo.get('topo', {}).get('properties', {}) + convergence_data = props.get('convergence_data', {}) + if props.get('topo_is_multi_vrf') and convergence_data.get('convergence_mapping'): + for primary, logical_names in convergence_data['convergence_mapping'].items(): + if peer_device in logical_names: + intf_mapping = convergence_data['converged_peers'][primary]['intf_mapping'] + return intf_mapping[peer_device]['orig_intf_map'][neighbor_intf] + + return neighbor_intf + + def __get_vm_peer_for_dut_intf(self, dut_intf): + """Return (vm_host, neighbor_intf_on_ceos, logical_peer_name) for a DUT interface.""" + peer_device = self.vm_neighbors[dut_intf]['name'] + neighbor_intf = self.vm_neighbors[dut_intf]['port'] + vm_host = self.nbrhosts[peer_device]['host'] + neighbor_intf = self.__resolve_vm_neighbor_intf(peer_device, neighbor_intf) + return vm_host, neighbor_intf, peer_device + def __verify_lag_lacp_timing(self, lacp_timer, exp_iface): if exp_iface is None: return @@ -166,7 +190,8 @@ def run_single_lag_lacp_rate_test(self, lag_name, lag_facts): neighbor_lag_intfs = [] for po_intf in po_interfaces: - neighbor_lag_intfs.append(self.vm_neighbors[po_intf]['port']) + port = self.vm_neighbors[po_intf]['port'] + neighbor_lag_intfs.append(self.__resolve_vm_neighbor_intf(peer_device, port)) try: lag_rate_current_setting = None @@ -211,10 +236,8 @@ def run_single_lag_test(self, lag_name, lag_facts): lag_facts, neighbor_intf, deselect_time=5) # Figure out remote VM and interface info for the lag member and run minlink test - peer_device = self.vm_neighbors[intf]['name'] - neighbor_intf = self.vm_neighbors[intf]['port'] - self.__verify_lag_minlink(self.nbrhosts[peer_device]['host'], lag_name, - lag_facts, neighbor_intf, deselect_time=95) + vm_host, neighbor_intf, _ = self.__get_vm_peer_for_dut_intf(intf) + self.__verify_lag_minlink(vm_host, lag_name, lag_facts, neighbor_intf, deselect_time=95) def run_lag_fallback_test(self, lag_name, lag_facts): logger.info("Start checking lag fall back for: %s" % lag_name) @@ -223,9 +246,7 @@ def run_lag_fallback_test(self, lag_name, lag_facts): po_fallback = lag_facts['lags'][lag_name]['po_config']['runner']['fallback'] # Figure out remote VM and interface info for the lag member and run lag fallback test - peer_device = self.vm_neighbors[intf]['name'] - neighbor_intf = self.vm_neighbors[intf]['port'] - vm_host = self.nbrhosts[peer_device]['host'] + vm_host, neighbor_intf, _ = self.__get_vm_peer_for_dut_intf(intf) wait_timeout = 120 delay = 5 diff --git a/tests/pc/test_po_update.py b/tests/pc/test_po_update.py index 6e5e4db7213..b84cdee6472 100644 --- a/tests/pc/test_po_update.py +++ b/tests/pc/test_po_update.py @@ -204,6 +204,22 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro or wait_until(10, 10, 0, pc_active, asichost, tmp_portchannel)) finally: # Recover all states + # The T1 topology golden config (smartswitch_t1.json) installs a static route + # "10.2.0.1/32 via 18.0.202.1" whose nexthop is resolved recursively. While + # PortChannel999 is operationally up and holds the 10.0.0.0/31 subnet, FRR + # resolves that static route through PortChannel999. sonic-utilities + # config/main.py checks "show ip route vrf all static" before removing the + # last IP on an interface and raises: + # "Cannot remove the last IP entry of interface PortChannel999. + # A static ip route is still bound to the RIF." + # Issuing admin-down on PortChannel999 first drops the LAG link, tears down + # BGP, and causes FRR to withdraw PortChannel999 from the recursive nexthop + # resolution. A brief sleep allows FRR to update its routing table before + # the IP removal is attempted, preserving the original cleanup order + # (IP removed before members). + if create_tmp_portchannel: + asichost.shutdown_interface(tmp_portchannel) + time.sleep(5) if add_tmp_portchannel_ip: asichost.config_ip_intf(tmp_portchannel, portchannel_ip + "/" + prefix_len, "remove") @@ -396,6 +412,22 @@ def del_add_members(): "Packets lost rate > {} during pc members add/removal, send_count: {}, match_count: {}".format( max_loss_rate, send_count, match_count)) finally: + # The T1 topology golden config (smartswitch_t1.json) installs a static route + # "10.2.0.1/32 via 18.0.202.1" whose nexthop is resolved recursively. While + # PortChannel999 is operationally up and holds the 10.0.0.0/31 subnet, FRR + # resolves that static route through PortChannel999. sonic-utilities + # config/main.py checks "show ip route vrf all static" before removing the + # last IP on an interface and raises: + # "Cannot remove the last IP entry of interface PortChannel999. + # A static ip route is still bound to the RIF." + # Issuing admin-down on PortChannel999 first drops the LAG link, tears down + # BGP, and causes FRR to withdraw PortChannel999 from the recursive nexthop + # resolution. A brief sleep allows FRR to update its routing table before + # the IP removal is attempted, preserving the original cleanup order + # (IP removed before members). + if create_tmp_pc: + asichost.shutdown_interface(tmp_pc) + time.sleep(5) if add_tmp_pc_ip: asichost.config_ip_intf(tmp_pc, pc_ip + "/31", "remove") wait_until(10, 2, 2, _check_ip_removed, asichost, tmp_pc) diff --git a/tests/pc/test_retry_count.py b/tests/pc/test_retry_count.py index 40b65851720..5fb935e56eb 100644 --- a/tests/pc/test_retry_count.py +++ b/tests/pc/test_retry_count.py @@ -75,7 +75,7 @@ def verify_retry_count(hosts, expected_retry_count): @pytest.fixture(scope="class") def higher_retry_count_on_peers(request, duthost, nbrhosts): - if request.config.getoption("neighbor_type") != "sonic": + if request.config.getoption("neighbor_type") not in ("sonic", "csonic"): pytest.skip("Only supported with SONiC neighbor") featureCheckResult = nbrhosts[list(nbrhosts.keys())[0]]['host'].command( @@ -102,7 +102,7 @@ def higher_retry_count_on_peers(request, duthost, nbrhosts): @pytest.fixture(scope="class") def higher_retry_count_on_dut(request, duthost, nbrhosts): - if request.config.getoption("neighbor_type") != "sonic": + if request.config.getoption("neighbor_type") not in ("sonic", "csonic"): pytest.skip("Only supported with SONiC neighbor") cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")["ansible_facts"] @@ -130,7 +130,7 @@ def higher_retry_count_on_dut(request, duthost, nbrhosts): @pytest.fixture(scope="function") -def config_reload_on_cleanup(request, nbrhosts, duthost): +def config_reload_on_cleanup(request, nbrhosts, duthost, loganalyzer): if request.config.getoption("enable_macsec"): pytest.skip("Skip for now, since config reload will disable macsec for future test cases") @@ -138,7 +138,7 @@ def config_reload_on_cleanup(request, nbrhosts, duthost): for nbr in list(nbrhosts.keys()): nbrhosts[nbr]['host'].command("sudo config reload -y") - config_reload(duthost, safe_reload=True) + config_reload(duthost, safe_reload=True, ignore_loganalyzer=loganalyzer) def log_lacpdu_packets(duthost, save_path): diff --git a/tests/pfcwd/test_pfcwd_all_port_storm.py b/tests/pfcwd/test_pfcwd_all_port_storm.py index c855c6a3782..f7ba0481186 100644 --- a/tests/pfcwd/test_pfcwd_all_port_storm.py +++ b/tests/pfcwd/test_pfcwd_all_port_storm.py @@ -209,7 +209,7 @@ class TestPfcwdAllPortStorm(object): PFC_RESTORE_THRESHOLD_PERCENTAGE = 100 def run_test(self, duthost, storm_hndle, expect_regex, syslog_marker, action, selected_test_ports, - stormed_ports_list=None, tbinfo=None): + stormed_ports_list=None, tbinfo=None, test_ports_info=None): """Storm generation/restoration on all ports and verification.""" loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=syslog_marker) ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages") @@ -244,8 +244,9 @@ def run_test(self, duthost, storm_hndle, expect_regex, syslog_marker, action, se timeout = max(timeout, 120) pytest_assert( wait_until(timeout, 2, 5, verify_all_ports_pfc_storm_in_expected_state, duthost, - storm_hndle, action, selected_test_ports, baseline_counters, threshold, - stormed_ports_list), + storm_hndle, action, selected_test_ports, + baseline_counters=baseline_counters, threshold_percentage=threshold, + stormed_ports_list=stormed_ports_list, test_ports_info=test_ports_info), f"Not enough ports reached {action} state (threshold: {threshold}%)" ) @@ -294,10 +295,14 @@ def test_all_port_storm_restore( action="storm", stormed_ports_list=stormed_ports_list, selected_test_ports=selected_test_ports, + test_ports_info=setup_pfc_test['test_ports'], tbinfo=tbinfo) logger.info(f"--- {len(stormed_ports_list)} ports entered storm state ---") logger.info("--- Testing if PFC storm is restored on stormed ports ---") + # test_ports_info is intentionally not passed during restore: the duplicate-neighbor + # adjustment in verify_all_ports_pfc_storm_in_expected_state only applies to the storm + # phase, so it has no effect here. self.run_test(duthost, storm_hndle, expect_regex=[EXPECT_PFC_WD_RESTORE_RE], syslog_marker="all_port_storm_restore", action="restore", stormed_ports_list=stormed_ports_list, diff --git a/tests/pfcwd/test_pfcwd_cli.py b/tests/pfcwd/test_pfcwd_cli.py index c5397be4201..e768d2344c7 100644 --- a/tests/pfcwd/test_pfcwd_cli.py +++ b/tests/pfcwd/test_pfcwd_cli.py @@ -6,7 +6,8 @@ from tests.common.fixtures.conn_graph_facts import enum_fanout_graph_facts # noqa: F401 from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.pfc_storm import PFCStorm -from tests.common.helpers.pfcwd_helper import start_wd_on_ports +from tests.common.helpers.pfcwd_helper import start_wd_on_ports, send_tx_egress, \ + shutdown_lag_members, restore_original_config from tests.common.helpers.pfcwd_helper import has_neighbor_device from tests.ptf_runner import ptf_runner from tests.common import constants @@ -15,8 +16,6 @@ from tests.common.helpers.pfcwd_helper import send_background_traffic, verify_pfc_storm_in_expected_state, parser_show_pfcwd_stat # noqa: E501 from tests.common.utilities import wait_until from tests.common.cisco_data import is_cisco_device -from tests.common import config_reload -from tests.common.devices.eos import EosHost pytestmark = [ pytest.mark.topology("t0", "t1", "lt2", "ft2") @@ -273,37 +272,6 @@ def get_lag_pkt_scale_factor(self): factor = 1.25 * num_dst_ports return factor - def send_tx_egress(self, action, verify): - """ - Send traffic with test port as the egress and verify if the packets get forwarded - or dropped based on the action - - Args: - action(string) : PTF test action - """ - logger.info("Check for egress {} on Tx port {}".format(action, self.pfc_wd_test_port)) - dst_port = "[" + str(self.pfc_wd_test_port_id) + "]" - if action == "forward" and type(self.pfc_wd_test_port_ids) == list: - dst_port = "".join(str(self.pfc_wd_test_port_ids)).replace(',', '') - ptf_params = {'router_mac': self.router_mac, - 'vlan_mac': self.vlan_mac, - 'queue_index': self.pfc_queue_index, - 'pkt_count': int(self.pfc_wd_test_pkt_count * self.get_lag_pkt_scale_factor()), - 'port_src': self.pfc_wd_rx_port_id[0], - 'port_dst': dst_port, - 'ip_dst': self.pfc_wd_test_neighbor_addr, - 'port_type': self.port_id_to_type_map[self.pfc_wd_rx_port_id[0]], - 'wd_action': action if verify else "dontcare", - 'ip_version': self.ip_version} - if self.pfc_wd_rx_port_vlan_id is not None: - ptf_params['port_src_vlan_id'] = self.pfc_wd_rx_port_vlan_id - if self.pfc_wd_test_port_vlan_id is not None: - ptf_params['port_dst_vlan_id'] = self.pfc_wd_test_port_vlan_id - log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) - ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, - log_file=log_file, is_python3=True) - def send_rx_ingress(self, action, verify): """ Send traffic with test port as the ingress and verify if the packets get forwarded @@ -337,73 +305,6 @@ def send_rx_ingress(self, action, verify): log_file=log_file, is_python3=True) -def _shutdown_lag_members(duthost, port, tbinfo, nbrhosts, port_type): - """Backs up config_db and modifies LAG members to isolate the selected port for PFCwd testing.""" - if port_type != 'portchannel': - return None, None, None - - config_facts = duthost.config_facts(host=duthost.hostname, source="persistent")['ansible_facts'] - portChannels = config_facts['PORTCHANNEL_MEMBER'] - portChannel = None - portChannelMembers = None - for intf in portChannels: - if port in portChannels[intf]: - portChannel = intf - portChannelMembers = portChannels[intf] - break - - dst_mgfacts = duthost.get_extended_minigraph_facts(tbinfo) - vm_neighbors = dst_mgfacts['minigraph_neighbors'] - peer_device = vm_neighbors[list(portChannelMembers.keys())[0]]['name'] - peer_port = vm_neighbors[list(portChannelMembers.keys())[0]]['port'] - vm_host = nbrhosts[peer_device]['host'] - neigh_port_channel = None - min_links = None - if isinstance(vm_host, EosHost): - neigh_port_channels = vm_host.eos_command( - commands=['show port-channel | json'])['stdout'][0]["portChannels"] - for po_name, po_config in neigh_port_channels.items(): - for member in po_config['activePorts']: - if member == peer_port: - neigh_port_channel = po_name - min_links = len(po_config['activePorts']) - break - - vm_host.eos_config(lines=['port-channel min-links 1'], parents=[f'int {neigh_port_channel}']) - - cmd_data = f'.PORTCHANNEL.{portChannel}.min_links = "1"' - - for member_port in portChannelMembers: - if member_port == port: - continue - cmd_data += f' | .PORT.{member_port}.admin_status="down"' - - cmd = f"""jq '{cmd_data}' /etc/sonic/config_db.json > /tmp/config_db.json""" - - _backup_original_config(duthost) - duthost.command(cmd, _uses_shell=True) - duthost.command("sudo cp /tmp/config_db.json /etc/sonic/config_db.json", _uses_shell=True) - config_reload(duthost, config_source='config_db', safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) - return vm_host, neigh_port_channel, min_links - - -def _backup_original_config(duthost): - """Backs up the current config_db.json before LAG modification.""" - duthost.command("cp /etc/sonic/config_db.json /tmp/config_db_backup.json", _uses_shell=True) - - -def _restore_original_config(duthost, port, vm_host, neigh_port_channel, min_links, port_type): - """Restores config_db and original LAG config after PFCwd testing.""" - if port_type != 'portchannel': - return - - if isinstance(vm_host, EosHost): - vm_host.eos_config(lines=[f'port-channel min-links {min_links}'], parents=[f'int {neigh_port_channel}']) - - duthost.command("sudo mv /tmp/config_db_backup.json /etc/sonic/config_db.json", _uses_shell=True) - config_reload(duthost, config_source='config_db', safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) - - @pytest.fixture(scope='function') def manage_lag_config(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, nbrhosts, setup_pfc_test): """Complete LAG config resource manager. @@ -415,28 +316,17 @@ def manage_lag_config(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinf duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] ports = setup_pfc_test['selected_test_ports'] port = list(ports.keys())[0] - port_type = ports[port]['test_port_type'] - vm_host, neigh_port_channel, min_links = _shutdown_lag_members( - duthost, port, tbinfo, nbrhosts, port_type) + vm_host, neigh_port_channel, min_links = shutdown_lag_members( + duthost, port, tbinfo, nbrhosts, ports) yield - _restore_original_config(duthost, port, vm_host, neigh_port_channel, min_links, port_type) + restore_original_config(duthost, port, vm_host, neigh_port_channel, min_links, ports) class TestPfcwdFunc(SetupPfcwdFunc): """ Test PFC function and supporting methods """ - def __shutdown_lag_members(self, duthost, selected_port, tbinfo, nbrhosts): - return _shutdown_lag_members( - duthost, selected_port, tbinfo, nbrhosts, - self.ports[selected_port]['test_port_type']) - - def __restore_original_config(self, duthost, selected_port, vm_host, neigh_port_channel, min_links): - _restore_original_config( - duthost, selected_port, vm_host, neigh_port_channel, min_links, - self.ports[selected_port]['test_port_type']) - def storm_detect_path(self, dut, port, action): """ Storm detection action and associated verifications @@ -517,10 +407,13 @@ def run_test(self, dut, port, action): ) # send traffic to egress port - self.traffic_inst.send_tx_egress(self.tx_action, False) + send_tx_egress(self.traffic_inst, self.tx_action, False, + pkt_count=int(self.traffic_inst.pfc_wd_test_pkt_count * + self.traffic_inst.get_lag_pkt_scale_factor())) time.sleep(10) # wait for the traffic to be processed pfcwd_stat_after_tx = parser_show_pfcwd_stat(dut, port, self.pfc_wd['queue_index']) logger.debug("pfcwd_stat_after_tx {}".format(pfcwd_stat_after_tx)) + if asic_type != 'vs': # check count, drop: tx_drop_count; forward: tx_ok_count if self.tx_action == "drop": diff --git a/tests/pfcwd/test_pfcwd_function.py b/tests/pfcwd/test_pfcwd_function.py index 1495846fc48..affce83538c 100644 --- a/tests/pfcwd/test_pfcwd_function.py +++ b/tests/pfcwd/test_pfcwd_function.py @@ -10,21 +10,28 @@ from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.helpers.pfc_storm import PFCStorm from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from tests.common.helpers.pfcwd_helper import start_wd_on_ports +from tests.common.helpers.pfcwd_helper import start_wd_on_ports, send_tx_egress from tests.common.helpers.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, \ fetch_vendor_specific_diagnosis_re from tests.common.helpers.pfcwd_helper import has_neighbor_device +from tests.common.helpers.pfcwd_helper import manage_lag_config # noqa: F401 from tests.ptf_runner import ptf_runner from tests.common import port_toggle from tests.common import constants from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled, dualtor_ports # noqa: F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa: F401, E501 -from tests.common.helpers.pfcwd_helper import send_background_traffic, check_pfc_storm_state +from tests.common.helpers.pfcwd_helper import send_background_traffic, check_pfc_storm_state, \ + verify_pfc_storm_in_expected_state, parser_show_pfcwd_stat, is_pfcwd_hw_recovery_enabled from tests.common.utilities import wait_until - +from tests.common.cisco_data import is_cisco_device +from tests.common.mellanox_data import is_mellanox_device PTF_PORT_MAPPING_MODE = 'use_orig_interface' +# Wait long enough for at least one pfcwd polling cycle to update counters +# between snapshots in run_pfcwd_storm_with_active_traffic. +PFCWD_POLL_WINDOW_SEC = 12 + TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") WD_ACTION_MSG_PFX = {"dontcare": "Verify PFCWD detection when queue buffer is not empty " "and proper function of pfcwd drop action", @@ -884,6 +891,7 @@ def test_pfcwd_actions( request, fake_storm, setup_pfc_test, + manage_lag_config, # noqa: F811 setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 ptfhost, @@ -975,7 +983,8 @@ def test_pfcwd_multi_port( self, request, fake_storm, - setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 + setup_pfc_test, manage_lag_config, # noqa: F811 + setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa: F811 toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, # noqa: F811 @@ -1068,7 +1077,8 @@ def test_pfcwd_mmu_change( self, request, fake_storm, - setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 + setup_pfc_test, manage_lag_config, # noqa: F811 + setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, dualtor_ports, # noqa: F811 setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa: F811 @@ -1170,7 +1180,8 @@ def test_pfcwd_mmu_change( def test_pfcwd_port_toggle( self, request, fake_storm, - setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 + setup_pfc_test, manage_lag_config, # noqa: F811 + setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 tbinfo, ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa: F811 toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, # noqa: F811 @@ -1279,8 +1290,145 @@ def test_pfcwd_port_toggle( logger.info("--- Stop PFCWD ---") self.dut.command("pfcwd stop") + def run_pfcwd_storm_with_active_traffic(self, dut, port, action): + restore_time = self.timers['pfc_wd_restore_time_large'] + # Hardware PFCwd platforms (e.g., Broadcom DNX) cap pfcwd + # restoration-time at 100-1000 ms; clamp to keep the CLI happy. + if is_pfcwd_hw_recovery_enabled(dut): + restore_time = min(restore_time, 1000) + detect_time = self.timers['pfc_wd_detect_time'] + queue = self.pfc_wd['queue_index'] + + start_wd_on_ports(dut, port, restore_time, detect_time, action) + send_tx_egress(self.traffic_inst, action, False, async_mode=True, pkt_count=5000000) + + try: + time.sleep(2) + self.storm_hndle.start_storm() + # Wait for storm detection + pytest_assert( + wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, + dut, port, queue, "storm"), + "PFC storm was not detected on port {} queue {}".format(port, queue)) + # Ensure storm is active → wait full polling window → snap + time.sleep(PFCWD_POLL_WINDOW_SEC) + wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, + dut, port, queue, "storm") + snap1 = parser_show_pfcwd_stat(dut, port, queue) + + # Again: confirm storm → wait poll window → snap + wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, + dut, port, queue, "storm") + time.sleep(PFCWD_POLL_WINDOW_SEC) + wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, + dut, port, queue, "storm") + snap2 = parser_show_pfcwd_stat(dut, port, queue) + + if dut.facts['asic_type'] != 'vs': + # On hardware-recovery PFCwd platforms (e.g., Broadcom DNX), the per-queue + # TX OK/DROP cells in 'show pfcwd stats' are sourced from the software- + # recovery code path and stay at 0 even when silicon is dropping. Storm + # detection is reported in both modes, so use storm_detect_count as the + # hardware-side signal. + if is_pfcwd_hw_recovery_enabled(dut): + counter = 'storm_detect_count' + else: + counter = 'tx_drop_count' if action == 'drop' else 'tx_ok_count' + val1 = int(snap1[0][counter]) + val2 = int(snap2[0][counter]) + logger.info("PFCWD storm traffic check: port={} queue={} counter={} snap1={} snap2={}".format( + port, queue, counter, val1, val2)) + pytest_assert(val1 > 0, "{} not incrementing after storm on port {} queue {}".format( + counter, port, queue)) + pytest_assert(val2 >= val1, "{} regressed during storm on port {} queue {}".format( + counter, port, queue)) + + finally: + self.ptf.shell("pkill -f 'pfc_wd.PfcWdTest'", module_ignore_errors=True) + self.storm_hndle.stop_storm() + pytest_assert( + wait_until(30, 2, 5, verify_pfc_storm_in_expected_state, + dut, port, queue, "restored"), + "PFC storm was not restored on port {} queue {}".format(port, queue)) + + def test_pfcwd_storm_during_traffic(self, request, setup_pfc_test, manage_lag_config, # noqa: F811 + setup_dut_test_params, enum_fanout_graph_facts, ptfhost, # noqa: F811 + duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts): + """ + Test pfcwd action when PFC storm is induced while traffic is already + flowing through the queue. Verifies via PTF dataplane checks that: + - Drop action: packets are dropped during storm + - Forward action: packets continue forwarding during storm + - Traffic resumes after storm restoration + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # 'show pfcwd stats' on Mellanox does not populate per-queue TX OK/DROP + # during storm even though RECOVERY_MECHANISM reports SOFTWARE; aligns + # with the legacy 'Pfcwd tests skipped on M* testbed' guard. + if is_mellanox_device(duthost): + pytest.skip("test_pfcwd_storm_during_traffic skipped on Mellanox") + setup_info = setup_pfc_test + setup_dut_info = setup_dut_test_params + ip_version = setup_info["ip_version"] + self.fanout_info = enum_fanout_graph_facts + self.ptf = ptfhost + self.dut = duthost + self.fanout = fanouthosts + self.timers = setup_info['pfc_timers'] + self.ports = setup_info['selected_test_ports'] + self.test_ports_info = setup_info['test_ports'] + if self.dut.topo_type == 't2': + key, value = list(self.ports.items())[0] + self.ports = {key: value} + self.neighbors = setup_info['neighbors'] + self.peer_dev_list = dict() + self.fake_storm = False + self.storm_hndle = None + self.rx_action = None + self.tx_action = None + self.is_dualtor = setup_dut_info['basicParams']['is_dualtor'] + self._bg_traffic_log_file = None + + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected") + + port = list(self.ports.keys())[0] + + logger.info("--- Testing pfcwd storm during traffic on {} ---".format(port)) + self.setup_test_params(port, setup_info['vlan'], init=True, ip_version=ip_version) + self.traffic_inst = SendVerifyTraffic( + self.ptf, + duthost.get_dut_iface_mac(self.pfc_wd['rx_port'][0]), + duthost.get_dut_iface_mac(self.pfc_wd['test_port']), + self.pfc_wd, + self.is_dualtor, + ip_version) + + if is_cisco_device(duthost): + actions = ['drop'] + else: + actions = ['drop', 'forward'] + + failures = [] + for action in actions: + logger.info("########## Pfcwd storm during traffic: port={} action={} ##########".format( + port, action)) + try: + self.set_traffic_action(duthost, action) + self.run_pfcwd_storm_with_active_traffic(self.dut, port, action) + except Exception as e: + logger.error("Action '{}' failed: {}".format(action, e)) + failures.append((action, str(e))) + finally: + if self.storm_hndle: + self.storm_hndle.stop_storm() + self.dut.command("pfcwd stop") + if failures: + pytest_assert(False, "Actions failed: {}".format(failures)) + def test_pfcwd_no_traffic( - self, request, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 + self, request, setup_pfc_test, manage_lag_config, # noqa: F811 + setup_dut_test_params, enum_fanout_graph_facts, # noqa: F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa: F811 toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, # noqa: F811 @@ -1298,6 +1446,7 @@ def test_pfcwd_no_traffic( duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] setup_info = setup_pfc_test setup_dut_info = setup_dut_test_params + ip_version = setup_info["ip_version"] self.fanout_info = enum_fanout_graph_facts self.ptf = ptfhost self.dut = duthost @@ -1325,7 +1474,7 @@ def test_pfcwd_no_traffic( for idx, port in enumerate(self.ports): logger.info("") logger.info("--- Testing non-Trafific Pfcwd actions on {} ---".format(port)) - self.setup_test_params(port, setup_info['vlan'], init=not idx) + self.setup_test_params(port, setup_info['vlan'], init=not idx, ip_version=ip_version) pfc_wd_restore_time_large = request.config.getoption("--restore-time") # wait time before we check the logs for the 'restore' signature. 'pfc_wd_restore_time_large' is in ms. diff --git a/tests/platform_tests/api/test_chassis.py b/tests/platform_tests/api/test_chassis.py index ba96a728244..1c8665fcd1f 100644 --- a/tests/platform_tests/api/test_chassis.py +++ b/tests/platform_tests/api/test_chassis.py @@ -349,7 +349,10 @@ def test_fans(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost, platf pytest.skip("No fans found on device") if duthost.facts.get("chassis"): - expected_num_fans = len(duthost.facts.get("chassis").get('fans')) + fans = duthost.facts.get("chassis").get('fans') + expected_num_fans = ( + sum(1 for f in fans if isinstance(f, dict) and f.get("name")) if fans else 0 + ) pytest_assert(num_fans == expected_num_fans, "Number of fans ({}) does not match expected number ({})" .format(num_fans, expected_num_fans)) diff --git a/tests/platform_tests/api/test_chassis_fans.py b/tests/platform_tests/api/test_chassis_fans.py index 84cf9af2876..86bd4a46d5e 100644 --- a/tests/platform_tests/api/test_chassis_fans.py +++ b/tests/platform_tests/api/test_chassis_fans.py @@ -4,9 +4,16 @@ import pytest from tests.common.helpers.platform_api import chassis, fan +from tests.common.helpers.platform_api.fan_discrete_speed_helper import ( + chassis_fan_speed_within_tolerance, + fan_speed_uses_platform_discrete_list, + get_chassis_fans_supported_speeds, + pick_initial_discrete_target_speed, +) from .platform_api_test_base import PlatformApiTestBase from tests.common.platform.device_utils import platform_api_conn, start_platform_api_service # noqa: F401 from tests.common.helpers.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon +from tests.common.utilities import wait_until ################################################### # TODO: Remove this after we transition to Python 3 @@ -190,10 +197,26 @@ def test_get_fans_target_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost, platform_api_conn): # noqa: F811 duthost = duthosts[enum_rand_one_per_hwsku_hostname] + use_discrete = fan_speed_uses_platform_discrete_list(duthost) + chassis_discrete_speeds = get_chassis_fans_supported_speeds(duthost) if use_discrete else None fans_skipped = 0 for i in range(self.num_fans): - speed_target_val = 25 + if use_discrete: + speed_target_val = pick_initial_discrete_target_speed( + self.num_fans, + chassis_discrete_speeds, + lambda p: self.get_fan_facts(duthost, p, True, "speed", "controllable"), + lambda p: self.get_fan_facts(duthost, p, 1, "speed", "minimum"), + lambda p: self.get_fan_facts(duthost, p, 100, "speed", "maximum"), + ) + if speed_target_val is None: + pytest.fail( + "Chassis fans: no controllable fan for discrete speed pick (platform {})".format( + duthost.facts.get("platform")) + ) + else: + speed_target_val = 25 speed_controllable = self.get_fan_facts(duthost, i, True, "speed", "controllable") if not speed_controllable: logger.info("test_get_fans_target_speed: Skipping chassis fan {} (speed not controllable)" @@ -204,7 +227,8 @@ def test_get_fans_target_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, speed_minimum = self.get_fan_facts(duthost, i, 25, "speed", "minimum") speed_maximum = self.get_fan_facts(duthost, i, 100, "speed", "maximum") if speed_minimum > speed_target_val or speed_maximum < speed_target_val: - speed_target_val = random.randint(speed_minimum, speed_maximum) + speed_target_val = self.get_fan_facts(duthost, i, random.randint(speed_minimum, speed_maximum), + "speed", "default") speed_set = fan.set_speed(platform_api_conn, i, speed_target_val) # noqa F841 # For x3b platform fan, the corresponding kernel driver (Max31790) ramps up the duty cycle to new value @@ -234,8 +258,29 @@ def test_set_fans_speed( fans_skipped = 0 duthost = duthosts[enum_rand_one_per_hwsku_hostname] + use_discrete = fan_speed_uses_platform_discrete_list(duthost) + chassis_discrete_speeds = get_chassis_fans_supported_speeds(duthost) if use_discrete else None + if use_discrete and not chassis_discrete_speeds: + pytest.fail( + "Discrete fan speed platforms require a `supported_speeds` field on an entry under " + "`chassis.fans` in platform.json (platform {})".format(duthost.facts.get("platform")) + ) + if duthost.facts["asic_type"] in ["cisco-8000"]: target_speed = random.randint(40, 60) + elif use_discrete: + target_speed = pick_initial_discrete_target_speed( + self.num_fans, + chassis_discrete_speeds, + lambda p: self.get_fan_facts(duthost, p, True, "speed", "controllable"), + lambda p: self.get_fan_facts(duthost, p, 1, "speed", "minimum"), + lambda p: self.get_fan_facts(duthost, p, 100, "speed", "maximum"), + ) + if target_speed is None: + pytest.fail( + "Chassis fans: no controllable fan for discrete speed pick (platform {})".format( + duthost.facts.get("platform")) + ) else: target_speed = random.randint(1, 100) @@ -248,24 +293,48 @@ def test_set_fans_speed( speed_minimum = self.get_fan_facts(duthost, i, 1, "speed", "minimum") speed_maximum = self.get_fan_facts(duthost, i, 100, "speed", "maximum") - if speed_minimum > target_speed or speed_maximum < target_speed: - target_speed = random.randint(speed_minimum, speed_maximum) + if use_discrete: + in_range = [s for s in chassis_discrete_speeds if speed_minimum <= s <= speed_maximum] + pick_from = in_range if in_range else chassis_discrete_speeds + if target_speed not in pick_from: + target_speed = random.choice(pick_from) + else: + if speed_minimum > target_speed or speed_maximum < target_speed: + target_speed = random.randint(speed_minimum, speed_maximum) speed = fan.get_speed(platform_api_conn, i) speed_delta = abs(speed-target_speed) speed_set = fan.set_speed(platform_api_conn, i, target_speed) # noqa: F841 - time_wait = 10 if speed_delta > 40 else 5 - time.sleep(self.get_fan_facts(duthost, i, time_wait, "speed", "delay")) + if use_discrete: + settled = wait_until( + 10, 2, 0, + chassis_fan_speed_within_tolerance, + platform_api_conn, i, + ) + act_speed = fan.get_speed(platform_api_conn, i) + self.expect( + settled, + "Chassis fan {} speed change from {} to {} did not settle within tolerance " + "within 10s (2s poll), actual speed {}".format(i, speed, target_speed, act_speed), + ) + else: + time_wait = 10 if speed_delta > 40 else 5 + time.sleep(self.get_fan_facts(duthost, i, time_wait, "speed", "delay")) - act_speed = fan.get_speed(platform_api_conn, i) - under_speed = fan.is_under_speed(platform_api_conn, i) - over_speed = fan.is_over_speed(platform_api_conn, i) - self.expect(not under_speed and not over_speed, - "Fan {} speed change from {} to {} is not within tolerance, actual speed {}" - .format(i, speed, target_speed, act_speed)) + act_speed = fan.get_speed(platform_api_conn, i) + under_speed = fan.is_under_speed(platform_api_conn, i) + over_speed = fan.is_over_speed(platform_api_conn, i) + self.expect(not under_speed and not over_speed, + "Fan {} speed change from {} to {} is not within tolerance, actual speed {}" + .format(i, speed, target_speed, act_speed)) if fans_skipped == self.num_fans: + if use_discrete: + pytest.skip( + "skipped as every chassis fan was skipped (not controllable, or no controllable fan " + "for discrete speed pick)" + ) pytest.skip("skipped as all chassis fans' speed is not controllable") self.assert_expectations() diff --git a/tests/platform_tests/api/test_fan_drawer_fans.py b/tests/platform_tests/api/test_fan_drawer_fans.py index b79cf90f4b9..918421cd2de 100644 --- a/tests/platform_tests/api/test_fan_drawer_fans.py +++ b/tests/platform_tests/api/test_fan_drawer_fans.py @@ -5,7 +5,14 @@ import pytest from tests.common.helpers.platform_api import chassis, fan_drawer, fan_drawer_fan +from tests.common.helpers.platform_api.fan_discrete_speed_helper import ( + fan_drawer_speed_within_tolerance, + fan_speed_uses_platform_discrete_list, + get_chassis_fans_supported_speeds, + pick_initial_discrete_target_speed, +) from tests.common.helpers.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon +from tests.common.utilities import wait_until from tests.common.platform.device_utils import platform_api_conn, start_platform_api_service # noqa: F401 from .platform_api_test_base import PlatformApiTestBase @@ -243,6 +250,8 @@ def test_get_fans_target_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, platform_api_conn, suspend_and_resume_hw_tc_on_mellanox_device): # noqa: F811 duthost = duthosts[enum_rand_one_per_hwsku_hostname] + use_discrete = fan_speed_uses_platform_discrete_list(duthost) + chassis_discrete_speeds = get_chassis_fans_supported_speeds(duthost) if use_discrete else None fan_drawers_skipped = 0 for j in range(self.num_fan_drawers): @@ -250,7 +259,21 @@ def test_get_fans_target_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, fans_skipped = 0 for i in range(num_fans): - speed_target_val = 25 + if use_discrete: + speed_target_val = pick_initial_discrete_target_speed( + num_fans, + chassis_discrete_speeds, + lambda p: self.get_fan_facts(duthost, j, p, True, "speed", "controllable"), + lambda p: self.get_fan_facts(duthost, j, p, 1, "speed", "minimum"), + lambda p: self.get_fan_facts(duthost, j, p, 100, "speed", "maximum"), + ) + if speed_target_val is None: + pytest.fail( + "Chassis fans: no controllable fan for discrete speed pick (platform {})".format( + duthost.facts.get("platform")) + ) + else: + speed_target_val = 25 speed_controllable = self.get_fan_facts(duthost, j, i, True, "speed", "controllable") if not speed_controllable: logger.info("test_get_fans_target_speed: Skipping fandrawer {} fan {} (speed not controllable)" @@ -261,7 +284,8 @@ def test_get_fans_target_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, speed_minimum = self.get_fan_facts(duthost, j, i, 25, "speed", "minimum") speed_maximum = self.get_fan_facts(duthost, j, i, 100, "speed", "maximum") if speed_minimum > speed_target_val or speed_maximum < speed_target_val: - speed_target_val = random.randint(speed_minimum, speed_maximum) + speed_target_val = self.get_fan_facts(duthost, j, i, random.randint(speed_minimum, speed_maximum), + "speed", "default") speed_set = fan_drawer_fan.set_speed(platform_api_conn, j, i, speed_target_val) # noqa F841 # For x3b platform fan, the corresponding kernel driver (Max31790) ramps up the duty cycle to new value @@ -297,12 +321,34 @@ def test_set_fans_speed( duthost = duthosts[enum_rand_one_per_hwsku_hostname] fan_drawers_skipped = 0 + use_discrete = fan_speed_uses_platform_discrete_list(duthost) + chassis_discrete_speeds = get_chassis_fans_supported_speeds(duthost) if use_discrete else None + if use_discrete and not chassis_discrete_speeds: + pytest.fail( + "Discrete fan speed platforms require a `supported_speeds` field on an entry under " + "`chassis.fans` in platform.json (platform {})".format(duthost.facts.get("platform")) + ) for j in range(self.num_fan_drawers): - target_speed = random.randint(1, 100) num_fans = fan_drawer.get_num_fans(platform_api_conn, j) fans_skipped = 0 + if use_discrete: + target_speed = pick_initial_discrete_target_speed( + num_fans, + chassis_discrete_speeds, + lambda p: self.get_fan_facts(duthost, j, p, True, "speed", "controllable"), + lambda p: self.get_fan_facts(duthost, j, p, 1, "speed", "minimum"), + lambda p: self.get_fan_facts(duthost, j, p, 100, "speed", "maximum"), + ) + if target_speed is None: + pytest.fail( + "Fan drawer {}: no controllable fan for discrete speed pick (platform {})".format( + j, duthost.facts.get("platform")) + ) + else: + target_speed = random.randint(1, 100) + for i in range(num_fans): speed_controllable = self.get_fan_facts(duthost, j, i, True, "speed", "controllable") if not speed_controllable: @@ -313,27 +359,52 @@ def test_set_fans_speed( speed_minimum = self.get_fan_facts(duthost, j, i, 1, "speed", "minimum") speed_maximum = self.get_fan_facts(duthost, j, i, 100, "speed", "maximum") - if speed_minimum > target_speed or speed_maximum < target_speed: - target_speed = random.randint(speed_minimum, speed_maximum) + if use_discrete: + in_range = [s for s in chassis_discrete_speeds if speed_minimum <= s <= speed_maximum] + pick_from = in_range if in_range else chassis_discrete_speeds + if target_speed not in pick_from: + target_speed = random.choice(pick_from) + else: + if speed_minimum > target_speed or speed_maximum < target_speed: + target_speed = random.randint(speed_minimum, speed_maximum) speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) speed_delta = abs(speed-target_speed) speed_set = fan_drawer_fan.set_speed(platform_api_conn, j, i, target_speed) # noqa: F841 - time_wait = 10 if speed_delta > 40 else 5 - time.sleep(self.get_fan_facts(duthost, j, i, time_wait, "speed", "delay")) + if use_discrete: + # Large steps can take longer than a fixed sleep; poll tolerance. + settled = wait_until( + 10, 2, 0, + fan_drawer_speed_within_tolerance, + platform_api_conn, j, i, + ) + act_speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) + self.expect( + settled, + "Fan drawer {} fan {} speed change from {} to {} did not settle within tolerance " + "within 10s (2s poll), actual speed {}".format(j, i, speed, target_speed, act_speed), + ) + else: + time_wait = 10 if speed_delta > 40 else 5 + time.sleep(self.get_fan_facts(duthost, j, i, time_wait, "speed", "delay")) - act_speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) - under_speed = fan_drawer_fan.is_under_speed(platform_api_conn, j, i) - over_speed = fan_drawer_fan.is_over_speed(platform_api_conn, j, i) - self.expect(not under_speed and not over_speed, - "Fan drawer {} fan {} speed change from {} to {} is not within tolerance, actual speed {}" - .format(j, i, speed, target_speed, act_speed)) + act_speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) + under_speed = fan_drawer_fan.is_under_speed(platform_api_conn, j, i) + over_speed = fan_drawer_fan.is_over_speed(platform_api_conn, j, i) + self.expect(not under_speed and not over_speed, + "Fan drawer {} fan {} speed change from {} to {} is not within tolerance, " + "actual speed {}".format(j, i, speed, target_speed, act_speed)) if fans_skipped == num_fans: fan_drawers_skipped += 1 if fan_drawers_skipped == self.num_fan_drawers: + if use_discrete: + pytest.skip( + "skipped as every fan drawer fan was skipped (not controllable, or no controllable fan " + "in a drawer for discrete speed pick)" + ) pytest.skip("skipped as all fandrawer fans' speed is not controllable") self.assert_expectations() diff --git a/tests/platform_tests/api/watchdog.yml b/tests/platform_tests/api/watchdog.yml index a8268deafa1..8d77b599b68 100644 --- a/tests/platform_tests/api/watchdog.yml +++ b/tests/platform_tests/api/watchdog.yml @@ -230,3 +230,10 @@ x86_64-nexthop_.*: valid_timeout: 10 greater_timeout: 16777 too_big_timeout: 16778 + +# th6-128 Nokia +x86_64-nokia_ixr7220_h6_128-r0: + default: + valid_timeout: 30 + greater_timeout: 170 + too_big_timeout: 350 diff --git a/tests/platform_tests/conftest.py b/tests/platform_tests/conftest.py index 8539e44b0bd..9f9dda8ba9b 100644 --- a/tests/platform_tests/conftest.py +++ b/tests/platform_tests/conftest.py @@ -12,6 +12,7 @@ from tests.common.platform.transceiver_utils import get_passive_cable_port_list, get_cmis_cable_ports_and_ver from tests.common.helpers.firmware_helper import PLATFORM_COMP_PATH_TEMPLATE from tests.common.platform.interface_utils import get_ports_with_flat_memory +from tests.common.helpers.platform_api import bmc logger = logging.getLogger(__name__) @@ -183,8 +184,9 @@ def check_pmon_uptime_minutes(duthost, minimal_runtime=6): def pytest_generate_tests(metafunc): val = metafunc.config.getoption('--fw-pkg') - if 'fw_pkg_name' in metafunc.fixturenames and val: - metafunc.parametrize('fw_pkg_name', val.split(','), scope="module") + if 'fw_pkg_name' in metafunc.fixturenames: + param_values = val.split(',') if val else [None] + metafunc.parametrize('fw_pkg_name', param_values, scope="module") if 'power_off_delay' in metafunc.fixturenames: delays = metafunc.config.getoption('power_off_delay') @@ -275,7 +277,11 @@ def cmis_cable_ports_and_ver(duthosts): @pytest.fixture(scope='module') -def fw_pkg(fw_pkg_name): +def fw_pkg(duthosts, enum_rand_one_per_hwsku_hostname, fw_pkg_name): + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + if not bmc.is_bmc_exists(duthost): + pytest.skip("BMC is not present, skipping BMC platform API tests") + if fw_pkg_name is None: pytest.skip("No fw package specified.") diff --git a/tests/platform_tests/daemon/conftest.py b/tests/platform_tests/daemon/conftest.py new file mode 100644 index 00000000000..47c5cda1c54 --- /dev/null +++ b/tests/platform_tests/daemon/conftest.py @@ -0,0 +1,31 @@ +"""Shared fixtures for the PMON daemon tests.""" +import time + +import pytest + + +def _get_daemon_duthost(request, duthosts): + """Resolve the DUT host for a daemon test module, honoring its `daemon_dut_hostname_fixture` override.""" + hostname_fixture = getattr(request.module, "daemon_dut_hostname_fixture", "rand_one_dut_hostname") + return duthosts[request.getfixturevalue(hostname_fixture)] + + +@pytest.fixture(scope="module") +def disable_pmon_container_autorestart(request, duthosts, disable_container_autorestart, enable_container_autorestart): + """Disable pmon container autorestart so a killed daemon respawns in place instead of restarting the container.""" + duthost = _get_daemon_duthost(request, duthosts) + daemon_name = request.module.daemon_name + disable_container_autorestart(duthost, testcase=daemon_name, feature_list=["pmon"]) + yield + enable_container_autorestart(duthost, testcase=daemon_name, feature_list=["pmon"]) + + +@pytest.fixture +def check_daemon_status(request, duthosts, disable_pmon_container_autorestart): + """Ensure the pmon daemon under test is running before the test starts.""" + duthost = _get_daemon_duthost(request, duthosts) + daemon_name = request.module.daemon_name + daemon_status, _ = duthost.get_pmon_daemon_status(daemon_name) + if daemon_status != "RUNNING": + duthost.start_pmon_daemon(daemon_name) + time.sleep(10) diff --git a/tests/platform_tests/daemon/test_chassisd.py b/tests/platform_tests/daemon/test_chassisd.py index ca125c0bb08..4f64b0fd7ac 100644 --- a/tests/platform_tests/daemon/test_chassisd.py +++ b/tests/platform_tests/daemon/test_chassisd.py @@ -29,6 +29,7 @@ expected_exited_status = "EXITED" daemon_name = "chassisd" +daemon_dut_hostname_fixture = "enum_rand_one_per_hwsku_hostname" SIG_STOP_SERVICE = None SIG_TERM = "-15" @@ -59,15 +60,6 @@ def teardown_module(duthosts, enum_rand_one_per_hwsku_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture -def check_daemon_status(duthosts, enum_rand_one_per_hwsku_hostname): - duthost = duthosts[enum_rand_one_per_hwsku_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def check_expected_daemon_status(duthost, expected_daemon_status): daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) return daemon_status == expected_daemon_status diff --git a/tests/platform_tests/daemon/test_fancontrol.py b/tests/platform_tests/daemon/test_fancontrol.py index 172f52a53c1..f59817cc2a6 100644 --- a/tests/platform_tests/daemon/test_fancontrol.py +++ b/tests/platform_tests/daemon/test_fancontrol.py @@ -28,6 +28,7 @@ expected_exited_status = "EXITED" daemon_name = "fancontrol" +daemon_dut_hostname_fixture = "enum_supervisor_dut_hostname" SIG_STOP_SERVICE = None SIG_TERM = "-15" @@ -56,15 +57,6 @@ def teardown_module(duthosts, enum_supervisor_dut_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture() -def check_daemon_status(duthosts, enum_supervisor_dut_hostname): - duthost = duthosts[enum_supervisor_dut_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def test_pmon_fancontrol_running_status(duthosts, enum_supervisor_dut_hostname): """ @summary: This test case is to check fancontrol status on dut diff --git a/tests/platform_tests/daemon/test_ledd.py b/tests/platform_tests/daemon/test_ledd.py index f3445d70f34..7d1039d7b42 100644 --- a/tests/platform_tests/daemon/test_ledd.py +++ b/tests/platform_tests/daemon/test_ledd.py @@ -57,15 +57,6 @@ def teardown_module(duthosts, rand_one_dut_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture() -def check_daemon_status(duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def check_expected_daemon_status(duthost, expected_daemon_status): daemon_status, _ = duthost.get_pmon_daemon_status(daemon_name) return daemon_status == expected_daemon_status diff --git a/tests/platform_tests/daemon/test_pcied.py b/tests/platform_tests/daemon/test_pcied.py index a2e90f3a8d7..6a051356bb8 100644 --- a/tests/platform_tests/daemon/test_pcied.py +++ b/tests/platform_tests/daemon/test_pcied.py @@ -77,15 +77,6 @@ def teardown_module(duthosts, rand_one_dut_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture -def check_daemon_status(duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def check_expected_daemon_status(duthost, expected_daemon_status): daemon_status, _ = duthost.get_pmon_daemon_status(daemon_name) return daemon_status == expected_daemon_status @@ -219,7 +210,7 @@ def test_pmon_pcied_term_and_start_status(check_daemon_status, duthosts, "{} status for SIG_TERM should not be {} with pid:{}!" .format(daemon_name, daemon_status, daemon_pid)) - time.sleep(10) + wait_until(120, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, diff --git a/tests/platform_tests/daemon/test_psud.py b/tests/platform_tests/daemon/test_psud.py index 0595843fd38..8ca89ac26af 100644 --- a/tests/platform_tests/daemon/test_psud.py +++ b/tests/platform_tests/daemon/test_psud.py @@ -30,6 +30,7 @@ expected_exited_status = "EXITED" daemon_name = "psud" +daemon_dut_hostname_fixture = "enum_supervisor_dut_hostname" SIG_STOP_SERVICE = None SIG_TERM = "-15" @@ -61,15 +62,6 @@ def teardown_module(duthosts, enum_supervisor_dut_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture -def check_daemon_status(duthosts, enum_supervisor_dut_hostname): - duthost = duthosts[enum_supervisor_dut_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def check_if_daemon_restarted(duthost, daemon_name, pre_daemon_pid): daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) return (daemon_pid > pre_daemon_pid) diff --git a/tests/platform_tests/daemon/test_syseepromd.py b/tests/platform_tests/daemon/test_syseepromd.py index e9567c2dcb9..d5d98df479c 100644 --- a/tests/platform_tests/daemon/test_syseepromd.py +++ b/tests/platform_tests/daemon/test_syseepromd.py @@ -60,15 +60,6 @@ def teardown_module(duthosts, rand_one_dut_hostname): check_critical_processes(duthost, watch_secs=10) -@pytest.fixture -def check_daemon_status(duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) - if daemon_status != "RUNNING": - duthost.start_pmon_daemon(daemon_name) - time.sleep(10) - - def check_expected_daemon_status(duthost, expected_daemon_status): daemon_status, _ = duthost.get_pmon_daemon_status(daemon_name) return daemon_status == expected_daemon_status diff --git a/tests/platform_tests/test_platform_info.py b/tests/platform_tests/test_platform_info.py index e24e8dd30c9..d1e181955c2 100644 --- a/tests/platform_tests/test_platform_info.py +++ b/tests/platform_tests/test_platform_info.py @@ -347,6 +347,24 @@ def test_turn_on_off_psu_and_check_psustatus(duthosts, enum_rand_one_per_hwsku_h # Group outlets/PDUs by PSU and toggle PDUs by PSU psu_to_pdus = get_grouped_pdus_by_psu(pdu_ctrl) + # Safety check: there must be at least 2 distinct PDU outlets across all PSUs, so that + # turning one PSU's outlet off still leaves another PSU powered. If every PSU hangs off + # a single shared outlet (e.g. both PSUs of a 2-PSU DUT on one outlet, as seen when the + # PDU connection graph maps PSU1 and PSU2 to the same outlet), turning it off removes + # all power from the DUT and reboots it + distinct_outlets = set() + for outlets in psu_to_pdus.values(): + for outlet in outlets: + distinct_outlets.add("{}/{}".format(outlet.get('pdu_name'), outlet.get('outlet_id'))) + if len(distinct_outlets) < 2: + logging.warning( + "All PSUs on %s share a single PDU outlet %s; turning it off would power off the " + "whole DUT. Please fix the PDU cabling/connection graph so each PSU has its own " + "outlet.", duthost.hostname, sorted(distinct_outlets)) + pytest_require( + len(distinct_outlets) >= 2, + "Skip the test: all PSUs on {} share PDU outlet(s) {}; toggling would power off the " + "whole DUT.".format(duthost.hostname, sorted(distinct_outlets))) # Get list of PSUs to skip from inventory configuration skip_psu_list = get_skip_mod_list(duthost, ['psus']) diff --git a/tests/platform_tests/test_sai_ocp_version.py b/tests/platform_tests/test_sai_ocp_version.py new file mode 100644 index 00000000000..c55138ca9b0 --- /dev/null +++ b/tests/platform_tests/test_sai_ocp_version.py @@ -0,0 +1,324 @@ +""" +Verify SAI versions on supported SONiC images. + +Background +---------- +Starting from SONiC build 20251110, different HwSKUs run different target +image types, each requiring specific BRCM SAI and OCP SAI versions: + + - "legacy-th" : Arista 7060CX family keeps old BRCM SAI paired with the + latest OCP SAI header. + +The test looks up the DUT's HwSKU in HWSKU_IMAGE_TYPE_MAP to determine which +image type (and therefore which expected versions) apply. If the HwSKU is not +present in the map the test is skipped. + +This test verifies (checks run in order; any unmet pre-condition is a clean +skip): + 1. DUT HwSKU is present in HWSKU_IMAGE_TYPE_MAP (skip otherwise) and maps to + an image type defined in IMAGE_TYPE_VERSIONS. + 2. SONiC OS build version is at or after the cut-off (20251110). + 3. ASIC vendor is Broadcom. + 4. Run `bcmcmd bsv` and parse BRCM SAI / OCP SAI / SDK versions. + 5. Compare parsed versions against the expected versions for the resolved + image type. +""" +import logging +import re + +import pytest + +from tests.common.helpers.assertions import pytest_assert, pytest_require + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.device_type('physical'), +] + +MIN_SONIC_BUILD_DATE = 20251110 + +# --------------------------------------------------------------------------- +# Image type definitions. +# Each entry maps an image-type name to the expected BRCM SAI and OCP SAI +# versions for that type. Add new image types here as the support matrix +# evolves. +# --------------------------------------------------------------------------- +IMAGE_TYPE_VERSIONS = { + # Arista 7060CX (Tomahawk) old BRCM SAI, latest OCP SAI header. + "legacy-th": { + "brcm_sai": "13.2.1.100", + "ocp_sai": "1.17.1", + } +} + +# --------------------------------------------------------------------------- +# HwSKU -> image-type mapping. +# Only HwSKUs listed here are tested; any other HwSKU causes the test to skip. +# Add HwSKUs and their corresponding image type as the support matrix grows. +# --------------------------------------------------------------------------- +HWSKU_IMAGE_TYPE_MAP = { + # --- legacy-th: Arista 7060CX (Tomahawk ASIC) --------------------------- + "Arista-7060CX-32S-C32": "legacy-th", + "Arista-7060CX-32S-D48C8": "legacy-th", + "Arista-7060CX-32S-Q32": "legacy-th" +} + +# `bcmcmd bsv` sample output: +# BRCM SAI ver: [13.2.1.100], OCP SAI ver: [1.17.1], SDK ver: [sdk-6.5.32-SP2] +# BRCM SAI cold boot ver:[13.2.1.100] +BSV_REGEX = re.compile( + r"BRCM SAI ver:\s*\[(?P[^\]]+)\]\s*,\s*" + r"OCP SAI ver:\s*\[(?P[^\]]+)\]\s*,\s*" + r"SDK ver:\s*\[(?P[^\]]+)\]" +) + + +RELEASE_VERSION_REGEX = re.compile(r"^(?:SONiC\.)?(20\d{6})\.\d+$") + +# Mainline internal / master builds, e.g. +# "SONiC.internal.167163159-dfd1f4ecaa0" +# "SONiC.master.1126962-04534d7ca" +# carry no YYYYMMDD date. They always track the latest code, so they are +# treated as satisfying the MIN_SONIC_BUILD_DATE cut-off. The trailing '.' +# after the keyword distinguishes them from the "internal-" release +# line form handled separately below. +MAINLINE_VERSION_REGEX = re.compile(r"^(?:SONiC\.)?(?:internal|master)\.", re.IGNORECASE) + +# Internal release-line builds, e.g. "SONiC.internal-202511". These map to a +# specific release branch identified by YYYYMM and run only when that release +# line is at or after the cut-off month (MIN_SONIC_BUILD_DATE // 100). +INTERNAL_RELEASE_REGEX = re.compile(r"^(?:SONiC\.)?internal-(\d{6})(?:\b|$)", re.IGNORECASE) + + +def _parse_build_date(os_version): + """Resolve a comparable YYYYMMDD build date from a SONiC version string. + + Accepted formats (as returned by `duthost.os_version` or shown in + `show version`): + + 1. Dated release -- real build date is parsed and returned: + "20251110.31" -> 20251110 + "SONiC.20251110.31" -> 20251110 + + 2. Mainline internal / master build -- no date; returned as + MIN_SONIC_BUILD_DATE so the cut-off gate always passes: + "SONiC.internal.167163159-dfd1f4ecaa0" + "SONiC.master.1126962-04534d7ca" + + 3. Internal release line "internal-" -- mapped to a month-level + comparable date (YYYYMM99) so the cut-off gate runs at month + granularity (e.g. internal-202511 -> 20251199 >= cut-off -> run; + internal-202505 -> 20250599 < cut-off -> skip): + "SONiC.internal-202511" + + Anything else (empty input, unrecognized format, etc.) returns None and + emits a warning, which causes the test to skip. + """ + if not os_version: + logger.warning("Empty SONiC version string; cannot parse build date") + return None + os_version = os_version.strip() + + # Case 2: mainline internal/master -- always at or ahead of the cut-off. + if MAINLINE_VERSION_REGEX.match(os_version): + logger.info( + "SONiC version '%s' is a mainline internal/master build; " + "treating build date as >= cut-off %s", + os_version, MIN_SONIC_BUILD_DATE, + ) + return MIN_SONIC_BUILD_DATE + + # Case 3: internal release line -- gate on the release month (YYYYMM). + match = INTERNAL_RELEASE_REGEX.match(os_version) + if match: + yyyymm = int(match.group(1)) + build_date = yyyymm * 100 + 99 + logger.info( + "SONiC version '%s' is internal release line %s; " + "comparable build date %s", + os_version, yyyymm, build_date, + ) + return build_date + + # Case 1: dated release build. + match = RELEASE_VERSION_REGEX.match(os_version) + if not match: + logger.warning( + "SONiC version '%s' is not a recognized version " + "(expected 'YYYYMMDD.', 'internal.<...>', 'master.<...>', " + "or 'internal-'); cannot parse build date", + os_version, + ) + return None + return int(match.group(1)) + + +def _parse_bsv_output(stdout): + """Parse `bcmcmd bsv` output and return dict {brcm_sai, ocp_sai, sdk}.""" + for line in stdout.splitlines(): + match = BSV_REGEX.search(line) + if match: + return match.groupdict() + return None + + +def _compare_sai_version(actual, expected): + """Compare BRCM SAI versions of the form 'A.B.C.D'. + + Rule: the first three components (major.minor.patch) must match exactly; + the last component (build) must be >= the expected build. + + Returns (ok: bool, reason: str). reason is "" when ok is True. + """ + try: + a_parts = [int(x) for x in actual.split(".")] + e_parts = [int(x) for x in expected.split(".")] + except (AttributeError, ValueError): + return False, "unparseable version (actual='{}', expected='{}')".format(actual, expected) + + if len(a_parts) != 4 or len(e_parts) != 4: + return False, "version must have 4 components (actual='{}', expected='{}')".format( + actual, expected) + + if a_parts[:3] != e_parts[:3]: + return False, "major.minor.patch differs: required '{}.x', got '{}'".format( + ".".join(str(p) for p in e_parts[:3]), actual) + + if a_parts[3] < e_parts[3]: + return False, "build component below minimum: required >= {}, got {}".format( + e_parts[3], a_parts[3]) + + return True, "" + + +def _compare_ocp_version(actual, expected): + """Compare OCP SAI header versions (e.g. '1.17.1'). + + Rule: actual must be >= expected, compared component-wise as integers. + + Returns (ok: bool, reason: str). reason is "" when ok is True. + """ + try: + a_parts = tuple(int(x) for x in actual.split(".")) + e_parts = tuple(int(x) for x in expected.split(".")) + except (AttributeError, ValueError): + return False, "unparseable version (actual='{}', expected='{}')".format(actual, expected) + + if a_parts < e_parts: + return False, "version below minimum: required >= '{}', got '{}'".format(expected, actual) + + return True, "" + + +def test_sai_ocp_version_per_hwsku(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """Verify BRCM SAI / OCP SAI version matches the image-type-based matrix. + + Sequence: + 1. Read HwSKU from DUT facts. + 2. Look up HwSKU in HWSKU_IMAGE_TYPE_MAP; skip if not present. Verify the + resolved image type exists in IMAGE_TYPE_VERSIONS (test-data sanity). + 3. Read SONiC OS build version; skip when older than MIN_SONIC_BUILD_DATE. + 4. Skip when ASIC vendor is not Broadcom. + 5. Run `bcmcmd bsv` and parse BRCM SAI / OCP SAI / SDK versions. + 6. Resolve expected BRCM SAI and OCP SAI versions from IMAGE_TYPE_VERSIONS. + 7. Assert parsed versions match the expected versions for the image type. + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + # ---- Step 1: Get HwSKU ------------------------------------------------- + hwsku = duthost.facts.get("hwsku", "") + logger.info("DUT %s HwSKU=%s", duthost.hostname, hwsku) + + # ---- Step 2: Resolve image type ---------------------------------------- + image_type = HWSKU_IMAGE_TYPE_MAP.get(hwsku) + pytest_require( + image_type is not None, + "HwSKU '{}' is not in HWSKU_IMAGE_TYPE_MAP, skipping".format(hwsku), + ) + logger.info("DUT %s HwSKU=%s resolved to image type '%s'", + duthost.hostname, hwsku, image_type) + pytest_assert( + image_type in IMAGE_TYPE_VERSIONS, + "Test-data inconsistency: HwSKU '{}' maps to image type '{}', which is " + "not defined in IMAGE_TYPE_VERSIONS (known types: {})".format( + hwsku, image_type, sorted(IMAGE_TYPE_VERSIONS.keys())), + ) + + # ---- Step 3: SONiC build version --------------------------------------- + os_version = duthost.os_version + build_date = _parse_build_date(os_version) + logger.info("DUT %s SONiC version: %s (parsed build date: %s)", + duthost.hostname, os_version, build_date) + + pytest_require( + build_date is not None, + "Cannot parse build date from SONiC version '{}', skipping".format(os_version), + ) + pytest_require( + build_date >= MIN_SONIC_BUILD_DATE, + "SONiC build {} is older than cut-off {}, skipping".format( + build_date, MIN_SONIC_BUILD_DATE), + ) + + # ---- Step 4: Vendor check ---------------------------------------------- + asic_type = duthost.facts.get("asic_type", "").lower() + pytest_require( + asic_type == "broadcom", + "ASIC type '{}' is not broadcom, this test is broadcom-only".format(asic_type), + ) + logger.info("DUT %s ASIC=%s", duthost.hostname, asic_type) + + # ---- Step 5: Read SAI / OCP / SDK versions from DUT ------------------- + result = duthost.shell("bcmcmd bsv", module_ignore_errors=True) + pytest_assert( + result.get("rc") == 0, + "`bcmcmd bsv` failed (rc={}): {}".format(result.get("rc"), result.get("stderr")), + ) + + parsed = _parse_bsv_output(result.get("stdout", "")) + pytest_assert( + parsed is not None, + "Could not parse `bcmcmd bsv` output:\n{}".format(result.get("stdout")), + ) + + brcm_sai = parsed["brcm_sai"] + ocp_sai = parsed["ocp_sai"] + sdk_ver = parsed["sdk"] + logger.info("DUT %s BRCM SAI=%s OCP SAI=%s SDK=%s", + duthost.hostname, brcm_sai, ocp_sai, sdk_ver) + + # ---- Step 6: Resolve required versions --------------------------------- + required_versions = IMAGE_TYPE_VERSIONS[image_type] + required_brcm_sai = required_versions.get("brcm_sai") + required_ocp_sai = required_versions.get("ocp_sai") + pytest_assert( + required_brcm_sai is not None and required_ocp_sai is not None, + "Test-data inconsistency: IMAGE_TYPE_VERSIONS['{}'] must define both " + "'brcm_sai' and 'ocp_sai' (got: {})".format(image_type, required_versions), + ) + logger.info( + "Required for image type '%s': BRCM SAI baseline=%s (same major.minor.patch, build >= %s), " + "OCP SAI header minimum=%s", + image_type, required_brcm_sai, required_brcm_sai.split(".")[-1], required_ocp_sai, + ) + + # ---- Step 7: Validate versions against the per-image-type baseline ----- + ok, reason = _compare_sai_version(brcm_sai, required_brcm_sai) + pytest_assert( + ok, + "BRCM SAI version check failed on {} (HwSKU={}, image_type={}): " + "required '{}' (same major.minor.patch, build >= {}), got '{}' -- {}".format( + duthost.hostname, hwsku, image_type, + required_brcm_sai, required_brcm_sai.split(".")[-1], + brcm_sai, reason), + ) + ok, reason = _compare_ocp_version(ocp_sai, required_ocp_sai) + pytest_assert( + ok, + "OCP SAI header version check failed on {} (HwSKU={}, image_type={}): " + "required >= '{}', got '{}' -- {}".format( + duthost.hostname, hwsku, image_type, + required_ocp_sai, ocp_sai, reason), + ) diff --git a/tests/qos/files/mellanox/qos_param_generator.py b/tests/qos/files/mellanox/qos_param_generator.py index c2b35422d93..fa25ba0c066 100644 --- a/tests/qos/files/mellanox/qos_param_generator.py +++ b/tests/qos/files/mellanox/qos_param_generator.py @@ -34,6 +34,11 @@ def __init__(self, qos_params, asic_type, speed_cable_len, dutConfig, ingressLos 'cell_size': 192, 'headroom_overhead': 47, 'private_headroom': 30 + }, + 'spc6': { + 'cell_size': 256, + 'headroom_overhead': 47, + 'private_headroom': 30 } } self.asic_type = asic_type @@ -65,6 +70,7 @@ def __init__(self, qos_params, asic_type, speed_cable_len, dutConfig, ingressLos self.src_asic_index = src_asic_index self.dst_dut_index = dst_dut_index self.dst_asic_index = dst_asic_index + self.qos_params_mlnx['descriptor_size'] = 32 if self.asic_type == "spc6" else 0 return def run(self): diff --git a/tests/qos/files/qos_params.th5.yaml b/tests/qos/files/qos_params.th5.yaml index 2f54597e461..6b175f2fad9 100644 --- a/tests/qos/files/qos_params.th5.yaml +++ b/tests/qos/files/qos_params.th5.yaml @@ -216,25 +216,24 @@ qos_params: topo-ft2-64: &topo-ft2-64 800000_5m: hdrm_pool_size: - dscps: - - 3 - - 4 + dscps: [3, 4] dst_port_id: 0 ecn: 1 margin: 4 - pgs: - - 3 - - 4 + pgs: [3, 4] pgs_num: 13 pkts_num_hdrm_full: 2853 pkts_num_hdrm_partial: 1124 - pkts_num_trig_pfc: 141861 + pkts_num_trig_pfc: 142285 + pkts_num_trig_pfc_multi: [142285, 71159, 35597, 17815, 8925, 4479, + 2257, 1145, 590, 312, 173, 103, 69] + src_port_ids: [1, 2, 3, 4, 5, 6, 7] lossy_queue_1: dscp: 8 ecn: 1 pg: 0 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 141835 + pkts_num_trig_egr_drp: 142259 pkts_num_egr_mem: 714 pkts_num_leak_out: 0 wm_pg_headroom: @@ -243,8 +242,8 @@ qos_params: ecn: 1 pg: 3 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 144715 - pkts_num_trig_pfc: 141861 + pkts_num_trig_ingr_drp: 145139 + pkts_num_trig_pfc: 142285 wm_pg_shared_lossless: cell_size: 254 dscp: 3 @@ -253,7 +252,7 @@ qos_params: pg: 3 pkts_num_fill_min: 34 pkts_num_margin: 4 - pkts_num_trig_pfc: 141861 + pkts_num_trig_pfc: 142285 wm_pg_shared_lossy: cell_size: 254 dscp: 8 @@ -262,14 +261,14 @@ qos_params: pg: 0 pkts_num_fill_min: 7 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 141835 + pkts_num_trig_egr_drp: 142259 wm_q_shared_lossless: cell_size: 254 dscp: 3 ecn: 1 pkts_num_fill_min: 0 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 144715 + pkts_num_trig_ingr_drp: 145139 queue: 3 wm_q_shared_lossy: cell_size: 254 @@ -277,38 +276,38 @@ qos_params: ecn: 1 pkts_num_fill_min: 7 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 141835 + pkts_num_trig_egr_drp: 142259 queue: 0 xoff_1: dscp: 3 ecn: 1 pg: 3 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 144715 - pkts_num_trig_pfc: 141861 + pkts_num_trig_ingr_drp: 145139 + pkts_num_trig_pfc: 142285 xoff_2: dscp: 4 ecn: 1 pg: 4 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 144715 - pkts_num_trig_pfc: 141861 + pkts_num_trig_ingr_drp: 145139 + pkts_num_trig_pfc: 142285 xon_1: dscp: 3 ecn: 1 pg: 3 pkts_num_dismiss_pfc: 14 pkts_num_margin: 4 - pkts_num_trig_pfc: 141861 + pkts_num_trig_pfc: 142285 xon_2: dscp: 4 ecn: 1 pg: 4 pkts_num_dismiss_pfc: 14 pkts_num_margin: 4 - pkts_num_trig_pfc: 141861 + pkts_num_trig_pfc: 142285 cell_size: 254 - hdrm_pool_wm_multiplier: 1 + hdrm_pool_wm_multiplier: 2 wrr: ecn: 1 limit: 80 @@ -337,25 +336,25 @@ qos_params: topo-lt2-p32o64: 800000_5m: hdrm_pool_size: - dscps: - - 3 - - 4 + dscps: [3, 4] dst_port_id: 0 ecn: 1 margin: 4 - pgs: - - 3 - - 4 + pgs: [3, 4] pgs_num: 24 pkts_num_hdrm_full: 2853 pkts_num_hdrm_partial: 701 - pkts_num_trig_pfc: 108469 + pkts_num_trig_pfc: 125589 + pkts_num_trig_pfc_multi: [125589, 62811, 31423, 15728, 7881, 3958, + 1996, 1015, 524, 279, 157, 95, 65, 49, 42, 38, 36, 35, 34, + 34, 34, 34, 34, 34] + src_port_ids: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] lossy_queue_1: dscp: 8 ecn: 1 pg: 0 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 108443 + pkts_num_trig_egr_drp: 125563 pkts_num_egr_mem: 714 pkts_num_leak_out: 0 wm_pg_headroom: @@ -364,8 +363,8 @@ qos_params: ecn: 1 pg: 3 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 111323 - pkts_num_trig_pfc: 108469 + pkts_num_trig_ingr_drp: 128443 + pkts_num_trig_pfc: 125589 wm_pg_shared_lossless: cell_size: 254 dscp: 3 @@ -374,7 +373,7 @@ qos_params: pg: 3 pkts_num_fill_min: 34 pkts_num_margin: 4 - pkts_num_trig_pfc: 108469 + pkts_num_trig_pfc: 125589 wm_pg_shared_lossy: cell_size: 254 dscp: 8 @@ -383,14 +382,14 @@ qos_params: pg: 0 pkts_num_fill_min: 7 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 108443 + pkts_num_trig_egr_drp: 125563 wm_q_shared_lossless: cell_size: 254 dscp: 3 ecn: 1 pkts_num_fill_min: 0 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 111323 + pkts_num_trig_ingr_drp: 128443 queue: 3 wm_q_shared_lossy: cell_size: 254 @@ -398,38 +397,38 @@ qos_params: ecn: 1 pkts_num_fill_min: 7 pkts_num_margin: 4 - pkts_num_trig_egr_drp: 108443 + pkts_num_trig_egr_drp: 125563 queue: 0 xoff_1: dscp: 3 ecn: 1 pg: 3 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 111323 - pkts_num_trig_pfc: 108469 + pkts_num_trig_ingr_drp: 128443 + pkts_num_trig_pfc: 125589 xoff_2: dscp: 4 ecn: 1 pg: 4 pkts_num_margin: 4 - pkts_num_trig_ingr_drp: 111323 - pkts_num_trig_pfc: 108469 + pkts_num_trig_ingr_drp: 128443 + pkts_num_trig_pfc: 125589 xon_1: dscp: 3 ecn: 1 pg: 3 pkts_num_dismiss_pfc: 14 pkts_num_margin: 4 - pkts_num_trig_pfc: 108469 + pkts_num_trig_pfc: 125589 xon_2: dscp: 4 ecn: 1 pg: 4 pkts_num_dismiss_pfc: 14 pkts_num_margin: 4 - pkts_num_trig_pfc: 108469 + pkts_num_trig_pfc: 125589 cell_size: 254 - hdrm_pool_wm_multiplier: 1 + hdrm_pool_wm_multiplier: 2 wrr: ecn: 1 limit: 80 diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index a4ddf93e3f8..fd824b146ca 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -51,7 +51,7 @@ class QosBase: "dualtor-120", "dualtor", "dualtor-64-breakout", "dualtor-aa", "dualtor-aa-56", "dualtor-aa-64-breakout", "t0-120", "t0-80", "t0-backend", "t0-56-o8v48", "t0-8-lag", "t0-standalone-32", "t0-standalone-64", "t0-standalone-128", "t0-standalone-256", "t0-28", "t0-isolated-d16u16s1", "t0-isolated-d16u16s2", - "t0-isolated-d96u32s2", "t0-isolated-d32u32s2", + "t0-isolated-d96u32s2", "t0-isolated-d32u32s2", "t0-isolated-d32u32s2-mix", "t0-88-o8c80", "t0-f2-d40u8", "t0-f2-d40u8-po2vlan" ] SUPPORTED_T1_TOPOS = ["t1", "t1-lag", "t1-64-lag", "t1-56-lag", "t1-backend", "t1-28-lag", "t1-32-lag", "t1-48-lag", @@ -61,7 +61,7 @@ class QosBase: "t1-isolated-d448u15-lag", "t1-isolated-v6-d448u15-lag"] SUPPORTED_PTF_TOPOS = ['ptf32', 'ptf64'] SUPPORTED_ASIC_LIST = ["pac", "gr", "gr2", "gb", "p200", "td2", "th", "th2", "spc1", "spc2", "spc3", "spc4", "spc5", - "td3", "th3", "j2c+", "jr2", "th5", "q3d"] + "spc6", "td3", "th3", "j2c+", "jr2", "th5", "q3d"] BREAKOUT_SKUS = ['Arista-7050-QX-32S'] LOW_SPEED_PORT_SKUS = ['Arista-7050CX3-32S-C28S4', 'Arista-7050CX3-32C-C28S4'] diff --git a/tests/qos/test_qos_dscp_mapping.py b/tests/qos/test_qos_dscp_mapping.py index f6fd7fdccf3..8305996c8f5 100644 --- a/tests/qos/test_qos_dscp_mapping.py +++ b/tests/qos/test_qos_dscp_mapping.py @@ -24,7 +24,8 @@ logger = logging.getLogger(__name__) pytestmark = [ - pytest.mark.topology('t0', 't1') + pytest.mark.topology('t0', 't1'), + pytest.mark.disable_memory_utilization ] DEFAULT_MAPPING_TYPE = "AZURE" @@ -559,7 +560,7 @@ def test_dscp_to_queue_mapping(self, ptfadapter, rand_selected_dut, localhost, d ): with allure.step("Do warm-reboot"): reboot(duthost, localhost, reboot_type="warm", safe_reboot=True, check_intf_up_ports=True, - wait_warmboot_finalizer=True) + wait_warmboot_finalizer=True, ignore_loganalyzer=loganalyzer) with allure.step("Run test after warm-reboot"): self._run_test(ptfadapter, duthost, tbinfo, test_params, inner_dst_ip_list, dut_qos_maps_module, diff --git a/tests/qos/test_qos_sai.py b/tests/qos/test_qos_sai.py index 51e18eef8cd..f6c3f756839 100644 --- a/tests/qos/test_qos_sai.py +++ b/tests/qos/test_qos_sai.py @@ -449,6 +449,8 @@ def testQosSaiPfcXoffLimit( if 'cell_size' in list(qosConfig[xoffProfile].keys()): testParams["cell_size"] = qosConfig[xoffProfile]["cell_size"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + self.runPtfTest( ptfhost, testCase="sai_qos_tests.PFCtest", testParams=testParams ) @@ -528,6 +530,8 @@ def testPfcStormWithSharedHeadroomOccupancy( if 'cell_size' in list(qosConfig[xonProfile].keys()): testParams["cell_size"] = qosConfig[xonProfile]["cell_size"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + # Params required for generating a PFC Storm duthost = dutConfig["srcDutInstance"] pfcwd_timers = set_pfc_timers() @@ -742,6 +746,8 @@ def testQosSaiPfcXonLimit( if 'cell_size' in list(qosConfig[xonProfile].keys()): testParams["cell_size"] = qosConfig[xonProfile]["cell_size"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + self.runPtfTest( ptfhost, testCase="sai_qos_tests.PFCXonTest", testParams=testParams ) @@ -950,6 +956,8 @@ def testQosSaiHeadroomPoolSize( if "pkts_num_trig_pfc_multi" in qosConfig["hdrm_pool_size"]: testParams.update({"pkts_num_trig_pfc_multi": qosConfig["hdrm_pool_size"]["pkts_num_trig_pfc_multi"]}) + self.set_test_params_descriptor_size(dutQosConfig, testParams) + if ('platform_asic' in dutTestParams["basicParams"] and dutTestParams["basicParams"]["platform_asic"] == "broadcom-dnx"): testParams['src_port_vlan'] = src_port_vlans @@ -1299,6 +1307,8 @@ def testQosSaiLossyQueue( if "pkts_num_margin" in list(qosConfig["lossy_queue_1"].keys()): testParams["pkts_num_margin"] = qosConfig["lossy_queue_1"]["pkts_num_margin"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + duthost = get_src_dst_asic_and_duts["src_dut"] if enable_lossy_pg_headroom: @@ -1813,6 +1823,8 @@ def testQosSaiPgSharedWatermark( if "internal_hdr_size" in list(qosConfig.keys()): testParams["internal_hdr_size"] = qosConfig["internal_hdr_size"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + self.runPtfTest( ptfhost, testCase="sai_qos_tests.PGSharedWatermarkTest", testParams=testParams @@ -1879,6 +1891,8 @@ def testQosSaiPgHeadroomWatermark( if "packet_size" in list(qosConfig["wm_pg_headroom"].keys()): testParams["packet_size"] = qosConfig["wm_pg_headroom"]["packet_size"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + self.runPtfTest( ptfhost, testCase="sai_qos_tests.PGHeadroomWatermarkTest", testParams=testParams @@ -2091,6 +2105,8 @@ def testQosSaiQSharedWatermark( if "pkts_num_margin" in list(qosConfig[queueProfile].keys()): testParams["pkts_num_margin"] = qosConfig[queueProfile]["pkts_num_margin"] + self.set_test_params_descriptor_size(dutQosConfig, testParams) + self.runPtfTest( ptfhost, testCase="sai_qos_tests.QSharedWatermarkTest", testParams=testParams @@ -2737,3 +2753,7 @@ def testQosSaiDscpEcn( self.check_and_set_ecn_status(duthost, qosConfig, 'on') elif ecn == "ecn_5": self.check_and_set_ecn_status(duthost, qosConfig, 'off') + + def set_test_params_descriptor_size(self, dutQosConfig, testParams): + if 'descriptor_size' in dutQosConfig["param"]: + testParams["descriptor_size"] = dutQosConfig["param"]["descriptor_size"] diff --git a/tests/redfish/__init__.py b/tests/redfish/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/redfish/conftest.py b/tests/redfish/conftest.py new file mode 100644 index 00000000000..34235ac7363 --- /dev/null +++ b/tests/redfish/conftest.py @@ -0,0 +1,383 @@ +import json +import logging +import os +import subprocess +import textwrap +import time + +import pytest + +from tests.common.helpers.assertions import pytest_require as pyrequire +from tests.common.utilities import wait_until +from tests.redfish.redfish_utils import BMC_TEST_CA_NAME, RedfishClient + +logger = logging.getLogger(__name__) + +REDFISH_ROOT = "/redfish/v1" +BMCWEB_CONTAINER = "redfish" + +BMCWEB_READY_TIMEOUT = 60 +BMCWEB_READY_POLL = 2 + + +@pytest.fixture(scope="session") +def bmc_duthost(duthosts, tbinfo): + """Return the SonicHost for the BMC under test. + + In the bmc-* topologies the testbed's DUT is the BMC itself -- the + ``-bmc`` inventory host, which runs SONiC -- so it is ``duts[0]``; + the host-side switch is a separate device referenced via the ``bmc_host`` + field. Skips the test if the resolved DUT is not a BMC. + """ + duthost = duthosts[tbinfo["duts"][0]] + pyrequire(duthost.is_bmc(), "Redfish BMC tests require a BMC DUT (NetworkBmc)") + return duthost + + +@pytest.fixture(scope="session") +def bmc_ip(bmc_duthost): + """Return the BMC management IP, used to build Redfish https URLs.""" + return bmc_duthost.mgmt_ip + + +@pytest.fixture(scope="session") +def redfish_base_url(bmc_ip): + return "https://{}{}".format(bmc_ip, REDFISH_ROOT) + + +@pytest.fixture(scope="session") +def redfish_client(bmc_ip, bmc_tls_certs): + """Return a RedfishClient configured for mTLS client-certificate auth. + + Depends on bmc_tls_certs so the BMC is in TLSStrict mode and the client + cert/key/CA paths are available before any Redfish request is issued. + """ + return RedfishClient( + bmc_ip, + bmc_tls_certs["cert"], + bmc_tls_certs["key"], + bmc_tls_certs["ca"], + ) + + +@pytest.fixture(scope="session") +def bmc_exec(bmc_duthost): + """Return a callable that runs a command on the BMC, returning (stdout, stderr, rc). + + Usage in tests: + stdout, stderr, rc = bmc_exec("docker exec redfish ls /etc/ssl/certs/https/") + + A non-zero exit is reported in rc rather than raised, so callers can assert on it. + """ + def _exec(cmd): + res = bmc_duthost.shell(cmd, module_ignore_errors=True) + return res["stdout"], res["stderr"], res["rc"] + return _exec + + +def _safe(fn, *args, **kwargs): + """Run a teardown step, log and swallow any exception so later steps still run.""" + try: + return fn(*args, **kwargs) + except Exception as e: + logger.warning("Teardown step %s(%s) failed: %s", + getattr(fn, "__name__", fn), args, e) + return None + + +def _run(cmd, cwd=None): + """Run a shell command locally inside the sonic-mgmt container.""" + subprocess.run(cmd, shell=True, check=True, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + +def _bmcweb_running(bmc_duthost): + """True iff `supervisorctl status bmcweb` reports RUNNING in the redfish container. + + Used as the wait_until condition. + """ + res = bmc_duthost.shell( + "docker exec {} supervisorctl status bmcweb".format(BMCWEB_CONTAINER), + module_ignore_errors=True, + ) + return res["rc"] == 0 and "RUNNING" in res["stdout"] + + +def _generate_ca_cert(cert_dir): + """Generate CA certificate and key.""" + d = str(cert_dir) + _run("openssl genrsa -out CA-key.pem 2048", cwd=d) + _run("openssl req -new -x509 -days 3650 -key CA-key.pem -out CA-cert.pem " + "-subj '/O=SONiC/OU=BMC/CN={}'".format(BMC_TEST_CA_NAME), + cwd=d) + + +def _generate_server_cert(cert_dir, bmc_ip): + """Generate server certificate signed by CA.""" + d = str(cert_dir) + _run("openssl genrsa -out server-key.pem 2048", cwd=d) + _run("openssl req -new -config openssl-server.cnf -key server-key.pem -out server.csr", + cwd=d) + _run("openssl x509 -req -extensions my_ext_section -extfile myext-server.cnf -days 730 " + "-in server.csr -CA CA-cert.pem -CAkey CA-key.pem -CAcreateserial " + "-out server-cert.pem", cwd=d) + # Create combined PEM (cert + key for bmcweb) + _run("cat server-cert.pem server-key.pem > server-combined.pem", cwd=d) + + +def _generate_client_cert(cert_dir, client_cn): + """Generate client certificate signed by CA.""" + d = str(cert_dir) + _run("openssl genrsa -out client-key.pem 2048", cwd=d) + _run("openssl req -new -config openssl-client.cnf -key client-key.pem -out client.csr", + cwd=d) + _run("openssl x509 -req -extensions my_ext_section -extfile myext-client.cnf -days 730 " + "-in client.csr -CA CA-cert.pem -CAkey CA-key.pem -CAcreateserial " + "-out client-cert.pem", cwd=d) + + +def _write_openssl_configs(cert_dir, bmc_ip, client_cn): + """Write OpenSSL config and extension files for server and client certs.""" + d = str(cert_dir) + configs = { + "openssl-server.cnf": textwrap.dedent("""\ + [ req ] + default_bits = 2048 + prompt = no + default_md = sha256 + distinguished_name = dn + req_extensions = v3_req + + [ dn ] + O = SONiC + OU = BMC + CN = {ip} + + [ v3_req ] + keyUsage = digitalSignature, keyAgreement + extendedKeyUsage = serverAuth + subjectAltName = @alt_names + + [ alt_names ] + DNS.1 = {ip} + IP.1 = {ip} + """.format(ip=bmc_ip)), + "myext-server.cnf": textwrap.dedent("""\ + [ my_ext_section ] + keyUsage = digitalSignature, keyAgreement + extendedKeyUsage = serverAuth + authorityKeyIdentifier = keyid + subjectKeyIdentifier = hash + subjectAltName = @alt_names + + [ alt_names ] + DNS.1 = {ip} + IP.1 = {ip} + """.format(ip=bmc_ip)), + "openssl-client.cnf": textwrap.dedent("""\ + [ req ] + default_bits = 2048 + prompt = no + default_md = sha256 + distinguished_name = dn + req_extensions = v3_req + + [ dn ] + O = SONiC + OU = BMC + CN = {cn} + + [ v3_req ] + keyUsage = digitalSignature + extendedKeyUsage = clientAuth + """.format(cn=client_cn)), + "myext-client.cnf": textwrap.dedent("""\ + [ my_ext_section ] + keyUsage = digitalSignature + extendedKeyUsage = clientAuth + authorityKeyIdentifier = keyid + subjectKeyIdentifier = hash + """), + } + for fname, content in configs.items(): + with open(os.path.join(d, fname), "w") as f: + f.write(content) + + +def _write_bmcweb_tls_config(cert_dir, tls_strict=True): + """Write bmcweb_tls_config.json with the given TLSStrict setting.""" + config = { + "auth_config": { + "BasicAuth": True, "Cookie": True, "SessionToken": True, + "XToken": True, "TLS": True, "TLSStrict": tls_strict, + "MTLSCommonNameParseMode": 2, + }, + "sessions": [], + "revision": 1, + } + with open(os.path.join(str(cert_dir), "bmcweb_tls_config.json"), "w") as f: + json.dump(config, f) + + +def _generate_certs(cert_dir, bmc_ip, client_cn): + """Generate CA, server, and client certificates using openssl CLI.""" + _write_openssl_configs(cert_dir, bmc_ip, client_cn) + _generate_ca_cert(cert_dir) + _generate_server_cert(cert_dir, bmc_ip) + _generate_client_cert(cert_dir, client_cn) + _write_bmcweb_tls_config(cert_dir, tls_strict=True) + + +@pytest.fixture(scope="session") +def bmc_clock_in_sync(bmc_duthost): + """Skip cert tests early if BMC clock is skewed beyond the cert NotBefore window. + + Generated certs use the sonic-mgmt container's current time as NotBefore. + If the BMC is behind that time, bmcweb sees the cert as not-yet-valid and + fails the TLS handshake with SSLV3_ALERT_BAD_CERTIFICATE — surfacing as an + opaque "bad certificate" error far from the actual cause. + """ + container_now = int(time.time()) + bmc_now = int(bmc_duthost.shell("date -u +%s")["stdout"].strip()) + skew = container_now - bmc_now + pyrequire( + abs(skew) <= 60, + "BMC clock is {}s {} sonic-mgmt container ({} vs {}). " + "Sync clocks before running cert tests.".format( + abs(skew), + "behind" if skew > 0 else "ahead of", + time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(container_now)), + time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(bmc_now)), + ), + ) + + +@pytest.fixture(scope="session") +def bmc_tls_certs(bmc_duthost, bmc_ip, bmc_clock_in_sync, tmp_path_factory): + """Generate TLS certificates, install them on the BMC, and clean up at session end. + + What this fixture does: + 1. Generates CA, server, and client certs inside the sonic-mgmt container using openssl. + 2. Copies the server cert, CA cert, and TLS config to the BMC. + 3. Installs the certs into the redfish container and enables TLSStrict in bmcweb. + 4. Yields a dict with paths to client-cert.pem, client-key.pem, and CA-cert.pem + for use in requests(cert=..., verify=...) calls. + 5. On teardown: removes the CA cert from the BMC truststore, restores TLSStrict=false, + and restarts bmcweb — leaving the BMC in Basic Auth mode as it was before. + """ + cert_dir = tmp_path_factory.mktemp("bmc_certs") + logger.info("Generating TLS certificates in {}".format(cert_dir)) + + # --- Step 1: Generate certificates using openssl inside the container --- + # Client cert CN must match a bmcweb user; use the "bmcweb" user. + _generate_certs(cert_dir, bmc_ip, client_cn="bmcweb") + + server_combined = str(cert_dir / "server-combined.pem") + ca_cert = str(cert_dir / "CA-cert.pem") + client_cert = str(cert_dir / "client-cert.pem") + client_key = str(cert_dir / "client-key.pem") + tls_config = str(cert_dir / "bmcweb_tls_config.json") + + logger.info("Certificates generated. Installing on BMC {}".format(bmc_ip)) + + # --- Step 2: Copy files to BMC --- + bmc_duthost.copy(src=server_combined, dest="/tmp/server-combined.pem") + bmc_duthost.copy(src=ca_cert, dest="/tmp/CA-cert.pem") + bmc_duthost.copy(src=tls_config, dest="/tmp/bmcweb_tls_config.json") + + # --- Step 3: Install server certificate (backup the original first) --- + bmc_duthost.shell( + "docker exec {} cp /etc/ssl/certs/https/server.pem " + "/etc/ssl/certs/https/server.pem.bak".format(BMCWEB_CONTAINER)) + bmc_duthost.shell( + "docker cp /tmp/server-combined.pem {}:/etc/ssl/certs/https/server.pem".format( + BMCWEB_CONTAINER)) + + # --- Step 4: Install CA certificate into truststore --- + bmc_duthost.shell( + "docker exec {} mkdir -p /etc/ssl/certs/authority".format(BMCWEB_CONTAINER)) + bmc_duthost.shell( + "docker cp /tmp/CA-cert.pem {}:/etc/ssl/certs/authority/CA-cert.pem".format( + BMCWEB_CONTAINER)) + + # Compute the hash on the BMC host (where /tmp/CA-cert.pem is accessible), + # then create the symlink inside the container using the explicit hash value. + # This avoids $() being evaluated in the wrong shell context. + ca_hash = bmc_duthost.shell( + "openssl x509 -hash -noout -in /tmp/CA-cert.pem")["stdout"].strip() + logger.info("CA cert hash: {}".format(ca_hash)) + bmc_duthost.shell( + 'docker exec {c} bash -c "cd /etc/ssl/certs/authority && ' + 'ln -sf CA-cert.pem {h}.0"'.format(c=BMCWEB_CONTAINER, h=ca_hash)) + + # --- Step 5: Enable TLSStrict and restart bmcweb --- + bmc_duthost.shell( + "docker exec {} supervisorctl stop bmcweb".format(BMCWEB_CONTAINER)) + bmc_duthost.shell( + "docker cp /tmp/bmcweb_tls_config.json {}:/bmcweb_persistent_data.json".format( + BMCWEB_CONTAINER)) + bmc_duthost.shell( + "docker exec {} supervisorctl start bmcweb".format(BMCWEB_CONTAINER)) + + # Wait for bmcweb to reach RUNNING again after the supervisorctl restart. + pyrequire( + wait_until(BMCWEB_READY_TIMEOUT, BMCWEB_READY_POLL, 0, + _bmcweb_running, bmc_duthost), + "bmcweb did not reach RUNNING within {}s after enabling TLSStrict".format( + BMCWEB_READY_TIMEOUT), + ) + logger.info("TLSStrict enabled. BMC is now in mTLS mode.") + + yield { + "cert": client_cert, + "key": client_key, + "ca": ca_cert, + "dir": str(cert_dir), + } + + # --- Teardown: restore BMC to Basic Auth mode --- + # Each step is wrapped in _safe() so a failure in one step doesn't leave + # the BMC half-configured (e.g. CA removed but server cert/TLSStrict not restored). + logger.info("Cleaning up: removing certs from BMC and disabling TLSStrict") + + _safe(bmc_duthost.shell, + "docker exec {} supervisorctl stop bmcweb".format(BMCWEB_CONTAINER), + module_ignore_errors=True) + + # Remove CA cert and its hash symlink from truststore. + # Compute hash on the host first (same reason as setup — avoid $() context issues). + ca_hash_res = _safe(bmc_duthost.shell, + "openssl x509 -hash -noout -in /tmp/CA-cert.pem 2>/dev/null", + module_ignore_errors=True) + if ca_hash_res and ca_hash_res["stdout"].strip(): + ca_hash_td = ca_hash_res["stdout"].strip() + _safe(bmc_duthost.shell, + 'docker exec {c} bash -c "rm -f /etc/ssl/certs/authority/CA-cert.pem ' + '/etc/ssl/certs/authority/{h}.0"'.format(c=BMCWEB_CONTAINER, h=ca_hash_td), + module_ignore_errors=True) + + # Restore the original server.pem from the backup taken at setup + _safe(bmc_duthost.shell, + 'docker exec {c} bash -c "mv -f /etc/ssl/certs/https/server.pem.bak ' + '/etc/ssl/certs/https/server.pem"'.format(c=BMCWEB_CONTAINER), + module_ignore_errors=True) + + # Write TLSStrict=false config, copy to BMC, install into container + _safe(_write_bmcweb_tls_config, cert_dir, tls_strict=False) + restore_config = str(cert_dir / "bmcweb_tls_config.json") + _safe(bmc_duthost.copy, src=restore_config, dest="/tmp/bmcweb_tls_restore.json") + _safe(bmc_duthost.shell, + "docker cp /tmp/bmcweb_tls_restore.json {}:/bmcweb_persistent_data.json".format( + BMCWEB_CONTAINER), + module_ignore_errors=True) + + _safe(bmc_duthost.shell, + "docker exec {} supervisorctl start bmcweb".format(BMCWEB_CONTAINER), + module_ignore_errors=True) + + # Wait for RUNNING again. + if not wait_until(BMCWEB_READY_TIMEOUT, BMCWEB_READY_POLL, 0, + _bmcweb_running, bmc_duthost): + logger.warning("bmcweb did not reach RUNNING within %ds during teardown", + BMCWEB_READY_TIMEOUT) + logger.info("BMC restored to Basic Auth mode.") diff --git a/tests/redfish/redfish_utils.py b/tests/redfish/redfish_utils.py new file mode 100644 index 00000000000..8a0f59c53bd --- /dev/null +++ b/tests/redfish/redfish_utils.py @@ -0,0 +1,93 @@ +""" +Shared Redfish test utilities for SONiC BMC Redfish API tests. +""" +import requests + +from tests.common.helpers.assertions import pytest_assert + + +BMC_TEST_CA_NAME = "SONiC BMC Test CA" + + +def redfish_url(bmc_ip, path): + """Build a full https URL for a Redfish path on the BMC.""" + return "https://{}{}".format(bmc_ip, path) + + +class RedfishClient: + """HTTP client for Redfish API calls using mTLS client-certificate auth.""" + + def __init__(self, bmc_ip, cert, key, ca, timeout=30): + self.base_url = "https://{}".format(bmc_ip) + self.cert = (cert, key) + self.verify = ca + self.timeout = timeout + + def _request(self, method, path, **kwargs): + return requests.request( + method, self.base_url + path, + cert=self.cert, verify=self.verify, timeout=self.timeout, + **kwargs, + ) + + def get(self, path, **kwargs): + return self._request("GET", path, **kwargs) + + def post(self, path, json=None, **kwargs): + return self._request("POST", path, json=json, **kwargs) + + def delete(self, path, **kwargs): + return self._request("DELETE", path, **kwargs) + + +def assert_field_equals(body, field, expected): + """Assert a top-level field equals an expected value.""" + actual = body.get(field, "") + pytest_assert( + actual == expected, + "Field '{}' must be {!r}, got: {!r}".format(field, expected, actual) + ) + + +def assert_field_contains(body, field, substring): + """Assert a top-level field contains a substring.""" + actual = body.get(field, "") + pytest_assert( + substring in actual, + "Field '{}' must contain {!r}, got: {!r}".format(field, substring, actual) + ) + + +def assert_field_nonempty(body, field): + """Assert a top-level field is a non-empty string.""" + actual = body.get(field, "") + pytest_assert( + isinstance(actual, str) and len(actual) > 0, + "Field '{}' must be a non-empty string, got: {!r}".format(field, actual) + ) + + +def assert_field_in(body, field, valid_values): + """Assert a top-level field is one of the valid values.""" + actual = body.get(field, "") + pytest_assert( + actual in valid_values, + "Field '{}' must be one of {}, got: {!r}".format(field, valid_values, actual) + ) + + +def assert_status_ok(response, path): + """Assert HTTP 200 from a given path.""" + pytest_assert( + response.status_code == 200, + "Expected HTTP 200 from {}, got: {}".format(path, response.status_code) + ) + + +def assert_member_count(body, minimum=1): + """Assert Members@odata.count >= minimum and Members array has entries.""" + count = body.get("Members@odata.count", 0) + pytest_assert( + count >= minimum, + "Members@odata.count must be >= {}, got: {}".format(minimum, count) + ) diff --git a/tests/redfish/test_redfish_cert_auth.py b/tests/redfish/test_redfish_cert_auth.py new file mode 100644 index 00000000000..104bca675ed --- /dev/null +++ b/tests/redfish/test_redfish_cert_auth.py @@ -0,0 +1,173 @@ +""" +Tests for Redfish certificate-based (mTLS) authentication. + +The bmc_tls_certs fixture (in conftest.py) runs before any test in this module: + - Generates CA, server, and client certificates locally + - Installs them on the BMC and enables TLSStrict in bmcweb + - Yields the client cert/key/CA paths for use in requests + - On teardown: removes the certs from BMC and restores Basic Auth mode +""" +import logging +import ssl +import pytest +import requests +import subprocess + +from tests.common.helpers.assertions import pytest_assert +from tests.redfish.redfish_utils import BMC_TEST_CA_NAME, assert_status_ok, redfish_url + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('bmc'), +] + +SERVICE_ROOT_PATH = "/redfish/v1" +UPDATE_SERVICE_PATH = "{}/UpdateService".format(SERVICE_ROOT_PATH) + + +class TestRedfishCertAuth: + + def test_cert_installed_on_bmc(self, bmc_tls_certs, bmc_exec): + """ + Verify certs are installed on the BMC. + + Checks that: + - The server cert exists at the expected container path + - The CA cert is present in the truststore + - bmcweb is running (supervisorctl shows RUNNING) + """ + # Server cert — verify it was signed by our CA (not the default self-signed) + stdout, _, _ = bmc_exec( + "docker exec redfish openssl x509 -in /etc/ssl/certs/https/server.pem -noout -issuer" + ) + pytest_assert( + BMC_TEST_CA_NAME in stdout, + "Server cert issuer must be our CA, got: {}".format(stdout) + ) + + # CA cert in truststore — verify subject matches our CA + stdout, _, _ = bmc_exec( + "docker exec redfish openssl x509 -in /etc/ssl/certs/authority/CA-cert.pem -noout -subject" + ) + pytest_assert( + BMC_TEST_CA_NAME in stdout, + "CA cert subject must contain our CA name, got: {}".format(stdout) + ) + + # bmcweb is running + stdout, _, _ = bmc_exec("docker exec redfish supervisorctl status bmcweb") + pytest_assert( + "RUNNING" in stdout, + "bmcweb is not running after cert install: {}".format(stdout) + ) + logger.info("All certs installed and bmcweb is RUNNING") + + def test_valid_cert_accepted(self, bmc_ip, bmc_tls_certs): + """ + Valid client certificate is accepted. + + GET /redfish/v1 with the generated client cert + key verified by our CA. + Must return HTTP 200. + """ + response = requests.get( + redfish_url(bmc_ip, SERVICE_ROOT_PATH), + cert=(bmc_tls_certs["cert"], bmc_tls_certs["key"]), + verify=bmc_tls_certs["ca"], + timeout=30, + ) + logger.info("GET {} (with cert) -> {}".format(SERVICE_ROOT_PATH, response.status_code)) + assert_status_ok(response, SERVICE_ROOT_PATH) + + def test_cert_auth_on_authenticated_endpoint(self, bmc_ip, bmc_tls_certs): + """ + Certificate-based auth works for an authenticated endpoint. + + GET /redfish/v1/UpdateService using only client cert (no Basic Auth). + Must return HTTP 200 with valid UpdateService data. + """ + response = requests.get( + redfish_url(bmc_ip, UPDATE_SERVICE_PATH), + cert=(bmc_tls_certs["cert"], bmc_tls_certs["key"]), + verify=bmc_tls_certs["ca"], + timeout=30, + ) + logger.info("GET {} (with cert) -> {}".format( + UPDATE_SERVICE_PATH, response.status_code)) + + assert_status_ok(response, UPDATE_SERVICE_PATH) + pytest_assert( + "@odata.id" in response.json(), + "Response missing @odata.id" + ) + + def test_no_cert_rejected(self, bmc_ip, bmc_tls_certs): + """ + Missing certificate is rejected when TLSStrict=true. + + GET /redfish/v1 with no client cert must fail with a TLS error + (TLSV13_ALERT_CERTIFICATE_REQUIRED) — not HTTP 200. + """ + tls_error_raised = False + try: + response = requests.get( + redfish_url(bmc_ip, SERVICE_ROOT_PATH), + verify=bmc_tls_certs["ca"], + timeout=30, + ) + # If we get here, the BMC did not require a cert — check it's at least not 200 + logger.warning("No TLS error raised — BMC may not be enforcing TLSStrict. " + "HTTP status: {}".format(response.status_code)) + pytest_assert( + response.status_code in (401, 403), + "Expected TLS error or HTTP 401/403 without client cert, got: {}".format( + response.status_code) + ) + except (requests.exceptions.SSLError, ssl.SSLError): + tls_error_raised = True + logger.info("TLS error raised as expected when no client cert is provided") + + if not tls_error_raised: + logger.info("BMC returned HTTP error (401/403) instead of TLS error — both are valid") + + def test_wrong_ca_rejected(self, bmc_ip, bmc_tls_certs, tmp_path): + """ + Certificate signed by an untrusted CA is rejected. + + Generates a fresh self-signed cert not signed by the BMC's trusted CA. + The request must fail with an SSL error or HTTP 401/403. + """ + + # Generate an untrusted self-signed cert on the fly + untrusted_key = str(tmp_path / "untrusted-key.pem") + untrusted_cert = str(tmp_path / "untrusted-cert.pem") + + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", untrusted_key, + "-out", untrusted_cert, + "-days", "1", + "-subj", "/CN=untrusted-client", + ], check=True, capture_output=True) + + tls_error_raised = False + try: + response = requests.get( + redfish_url(bmc_ip, SERVICE_ROOT_PATH), + cert=(untrusted_cert, untrusted_key), + verify=bmc_tls_certs["ca"], + timeout=30, + ) + logger.warning("No TLS error raised for untrusted cert — HTTP status: {}".format( + response.status_code)) + pytest_assert( + response.status_code in (401, 403), + "Expected TLS error or HTTP 401/403 for untrusted cert, got: {}".format( + response.status_code) + ) + except (requests.exceptions.SSLError, ssl.SSLError): + tls_error_raised = True + logger.info("TLS error raised as expected for untrusted client cert") + + if not tls_error_raised: + logger.info("BMC returned HTTP error (401/403) for untrusted cert — both are valid") diff --git a/tests/redfish/test_redfish_computer_reset.py b/tests/redfish/test_redfish_computer_reset.py new file mode 100644 index 00000000000..fc2494b65d3 --- /dev/null +++ b/tests/redfish/test_redfish_computer_reset.py @@ -0,0 +1,232 @@ +""" +Tests for Redfish ComputerSystem.Reset action endpoint. + +WARNING: GracefulShutdown and PowerCycle tests trigger actual power state changes +on the BMC DUT. They restore the system to its original power state after each test. +""" +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('bmc'), +] + +RESET_PATH = "/redfish/v1/Systems/system/Actions/ComputerSystem.Reset" + +POWER_ON_TIMEOUT = 120 # seconds to wait for x86 CPU to come out of reset +POWER_OFF_TIMEOUT = 120 # seconds to wait for x86 CPU to be held in reset +POLL_INTERVAL = 5 # seconds between CPU-state polls + +POWER_CYCLE_OFF_TIMEOUT = 30 +POWER_CYCLE_OFF_POLL = 1 + +CPU_STATUS_CMD = "sudo switch_cpu_utils.sh status" + + +def _cpu_running(bmc_exec): + """Return True iff the x86 host CPU is OUT OF RESET (running). + + Trusts the BMC-side switch_cpu_utils.sh output (which reads the hardware + reset pin). + """ + stdout, _, _ = bmc_exec(CPU_STATUS_CMD) + return "OUT OF RESET" in stdout + + +def _cpu_state_matches(bmc_exec, want_running): + """wait_until predicate: True iff the CPU running state matches want_running.""" + running = _cpu_running(bmc_exec) + logger.info("CPU running=%s (waiting for running=%s)", running, want_running) + return running == want_running + + +def _ensure_system_on(redfish_client, bmc_exec): + """Power on the x86 CPU if it is not currently running.""" + if _cpu_running(bmc_exec): + return + logger.info("CPU is in reset, sending ResetType=On to restore") + redfish_client.post(RESET_PATH, json={"ResetType": "On"}) + pytest_assert( + wait_until(POWER_ON_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, True), + "x86 CPU did not come out of reset within {}s".format(POWER_ON_TIMEOUT), + ) + + +def _ensure_system_in_reset(redfish_client, bmc_exec): + """Hold the x86 CPU in reset if it is currently running.""" + if not _cpu_running(bmc_exec): + return + logger.info("CPU is running, sending ResetType=GracefulShutdown to enter reset") + redfish_client.post(RESET_PATH, json={"ResetType": "GracefulShutdown"}) + pytest_assert( + wait_until(POWER_OFF_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, False), + "x86 CPU did not enter reset within {}s".format(POWER_OFF_TIMEOUT), + ) + + +class TestRedfishComputerReset: + + @pytest.fixture(autouse=True) + def _restore_cpu_on(self, redfish_client, bmc_exec): + """Best-effort finalizer: never leave the x86 CPU held in reset. + + GracefulShutdown / PowerCycle power the CPU off and restore it before + returning, but a mid-test failure (failed assertion, timeout) would + otherwise leave the CPU in reset for every subsequent test. This runs + after each test and powers the CPU back on if it is still in reset, + logging instead of asserting so it never masks the test's own failure. + """ + yield + if _cpu_running(bmc_exec): + return + logger.warning("CPU left in reset after test; restoring with ResetType=On") + redfish_client.post(RESET_PATH, json={"ResetType": "On"}) + if not wait_until(POWER_ON_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, True): + logger.error("Failed to restore x86 CPU to running state in teardown") + + def test_reset_on_when_already_on(self, redfish_client, bmc_exec): + """ + ResetType=On when the CPU is already running is a no-op. + + Brings the CPU to a running state first, then POST ResetType=On and + verify the BMC accepts the request and the CPU stays running. + """ + _ensure_system_on(redfish_client, bmc_exec) + + response = redfish_client.post(RESET_PATH, json={"ResetType": "On"}) + logger.info("POST {} ResetType=On -> {}".format(RESET_PATH, response.status_code)) + + pytest_assert( + response.status_code in (200, 204), + "Expected HTTP 200 or 204, got: {}".format(response.status_code) + ) + + pytest_assert( + _cpu_running(bmc_exec), + "x86 CPU should remain running after ResetType=On from a running state" + ) + + def test_reset_on_when_in_reset(self, redfish_client, bmc_exec): + """ + ResetType=On brings the CPU out of reset. + + Holds the CPU in reset first, then POST ResetType=On and verify the + CPU transitions to OUT OF RESET (running). + """ + _ensure_system_in_reset(redfish_client, bmc_exec) + + response = redfish_client.post(RESET_PATH, json={"ResetType": "On"}) + logger.info("POST {} ResetType=On -> {}".format(RESET_PATH, response.status_code)) + + pytest_assert( + response.status_code in (200, 204), + "Expected HTTP 200 or 204, got: {}".format(response.status_code) + ) + + reached = wait_until(POWER_ON_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, True) + pytest_assert(reached, "x86 CPU did not come out of reset within {}s".format( + POWER_ON_TIMEOUT)) + + def test_reset_graceful_shutdown(self, redfish_client, bmc_exec): + """ + Reset with valid ResetType "GracefulShutdown". + + Verifies the x86 CPU is held in reset after graceful shutdown, then + restores it to running. + """ + _ensure_system_on(redfish_client, bmc_exec) + + response = redfish_client.post(RESET_PATH, json={"ResetType": "GracefulShutdown"}) + logger.info("POST {} ResetType=GracefulShutdown -> {}".format( + RESET_PATH, response.status_code)) + + pytest_assert( + response.status_code in (200, 204), + "Expected HTTP 200 or 204, got: {}".format(response.status_code) + ) + + reached = wait_until(POWER_OFF_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, False) + pytest_assert(reached, "x86 CPU was not held in reset within {}s".format( + POWER_OFF_TIMEOUT)) + + _ensure_system_on(redfish_client, bmc_exec) + + def test_reset_power_cycle(self, redfish_client, bmc_exec): + """ + Reset with valid ResetType "PowerCycle". + + Observes BOTH transitions — CPU enters reset, then exits reset — so + the test cannot pass trivially if the BMC silently no-ops the API and + leaves the CPU running the whole time. + """ + _ensure_system_on(redfish_client, bmc_exec) + + response = redfish_client.post(RESET_PATH, json={"ResetType": "PowerCycle"}) + logger.info("POST {} ResetType=PowerCycle -> {}".format(RESET_PATH, response.status_code)) + + pytest_assert( + response.status_code in (200, 204), + "Expected HTTP 200 or 204, got: {}".format(response.status_code) + ) + + # First observe the off-transition. The off-window is brief (~1-2s), + # so poll faster than POLL_INTERVAL to avoid missing it. + entered_reset = wait_until(POWER_CYCLE_OFF_TIMEOUT, POWER_CYCLE_OFF_POLL, 0, + _cpu_state_matches, bmc_exec, False) + pytest_assert( + entered_reset, + "x86 CPU did not enter reset after PowerCycle within {}s — " + "BMC may have silently no-op'd the API".format(POWER_CYCLE_OFF_TIMEOUT), + ) + + # Then wait for it to come back out. + reached = wait_until(POWER_ON_TIMEOUT, POLL_INTERVAL, 0, + _cpu_state_matches, bmc_exec, True) + pytest_assert(reached, + "x86 CPU did not return to OUT OF RESET after PowerCycle within {}s".format( + POWER_ON_TIMEOUT)) + + def test_reset_invalid_type(self, redfish_client): + """ + Reset with invalid ResetType is rejected. + + POST ResetType=InvalidType must return HTTP 400 with a Redfish error body. + """ + response = redfish_client.post(RESET_PATH, json={"ResetType": "InvalidType"}) + logger.info("POST {} ResetType=InvalidType -> {}".format(RESET_PATH, response.status_code)) + + pytest_assert( + response.status_code == 400, + "Expected HTTP 400 for invalid ResetType, got: {}".format(response.status_code) + ) + + try: + error_body = response.json() + except ValueError: + error_body = None + pytest_assert( + isinstance(error_body, dict), + "Error response is not a valid JSON object: {}".format(response.text) + ) + + # Redfish error responses carry an "error" object with at least a + # "code" and a "message" field (DSP0266 error payload shape). + error = (error_body or {}).get("error") + pytest_assert( + isinstance(error, dict), + "Expected a Redfish error object under 'error', got: {}".format(error_body) + ) + pytest_assert( + "code" in (error or {}) and "message" in (error or {}), + "Redfish error object must contain 'code' and 'message', got: {}".format(error) + ) diff --git a/tests/redfish/test_redfish_firmware_inventory.py b/tests/redfish/test_redfish_firmware_inventory.py new file mode 100644 index 00000000000..18d37332f17 --- /dev/null +++ b/tests/redfish/test_redfish_firmware_inventory.py @@ -0,0 +1,155 @@ +""" +Tests for Redfish Firmware Inventory endpoints. +""" +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.redfish.redfish_utils import ( + assert_field_contains, assert_field_equals, assert_field_nonempty, + assert_member_count, assert_status_ok, +) + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('bmc'), +] + +FIRMWARE_COLLECTION_PATH = "/redfish/v1/UpdateService/FirmwareInventory" +BMC_FIRMWARE_PATH = "{}/bmc".format(FIRMWARE_COLLECTION_PATH) +BIOS_FIRMWARE_PATH = "{}/bios".format(FIRMWARE_COLLECTION_PATH) +SWITCH_FIRMWARE_PATH = "{}/switch".format(FIRMWARE_COLLECTION_PATH) + +EXPECTED_MEMBERS = {"bmc", "bios", "switch"} + + +def _assert_software_inventory_shape(body, component_id): + """Schema invariants shared by every SoftwareInventory entry.""" + assert_field_equals(body, "Id", component_id) + assert_field_contains(body, "@odata.type", "SoftwareInventory") + assert_field_equals(body, "Name", "Software Inventory") + + status = body.get("Status", {}) + pytest_assert( + status.get("State") == "Enabled", + "Status.State must be 'Enabled', got: {!r}".format(status.get("State")) + ) + pytest_assert( + status.get("Health") == "OK", + "Status.Health must be 'OK', got: {!r}".format(status.get("Health")) + ) + pytest_assert( + status.get("HealthRollup") == "OK", + "Status.HealthRollup must be 'OK', got: {!r}".format(status.get("HealthRollup")) + ) + + pytest_assert( + isinstance(body.get("Updateable"), bool), + "Updateable must be a boolean, got: {!r}".format(body.get("Updateable")) + ) + assert_field_nonempty(body, "Version") + + +class TestRedfishFirmwareInventory: + + def test_firmware_inventory_collection(self, redfish_client): + """ + Firmware inventory collection. + + GET /redfish/v1/UpdateService/FirmwareInventory must return HTTP 200 + with at least one member and expected firmware components. + """ + response = redfish_client.get(FIRMWARE_COLLECTION_PATH) + logger.info("GET {} -> {}".format(FIRMWARE_COLLECTION_PATH, response.status_code)) + + assert_status_ok(response, FIRMWARE_COLLECTION_PATH) + + body = response.json() + assert_member_count(body) + + members = body.get("Members", []) + member_ids = {m.get("@odata.id", "").split("/")[-1] for m in members} + logger.info("Firmware members: {}".format(member_ids)) + + missing = EXPECTED_MEMBERS - member_ids + pytest_assert( + not missing, + "Expected firmware members {} not found in collection. Present: {}".format( + missing, member_ids) + ) + + def test_firmware_bmc(self, redfish_client): + """ + BMC firmware inventory entry. + + GET /redfish/v1/UpdateService/FirmwareInventory/bmc — validates the BMC + SoftwareInventory entry has a real build version and links back to + /redfish/v1/Managers/bmc via RelatedItem. + """ + response = redfish_client.get(BMC_FIRMWARE_PATH) + logger.info("GET {} -> {}".format(BMC_FIRMWARE_PATH, response.status_code)) + assert_status_ok(response, BMC_FIRMWARE_PATH) + + body = response.json() + assert_field_equals(body, "@odata.id", BMC_FIRMWARE_PATH) + _assert_software_inventory_shape(body, "bmc") + assert_field_equals(body, "Description", "BMC image") + + # BMC must report a real build string, not the "N/A" placeholder. + version = body.get("Version") + pytest_assert( + version != "N/A", + "BMC Version must be a real build string, got: {!r}".format(version) + ) + logger.info("BMC Version: {!r}".format(version)) + + related_ids = [r.get("@odata.id") for r in body.get("RelatedItem", [])] + pytest_assert( + "/redfish/v1/Managers/bmc" in related_ids, + "BMC RelatedItem must contain '/redfish/v1/Managers/bmc', got: {}".format( + related_ids) + ) + + def test_firmware_bios(self, redfish_client): + """ + BIOS firmware inventory entry. + + GET /redfish/v1/UpdateService/FirmwareInventory/bios — validates the BIOS + SoftwareInventory entry. Version is currently "N/A" and there is no + RelatedItem in this BMC build, so only schema shape is asserted. + """ + response = redfish_client.get(BIOS_FIRMWARE_PATH) + logger.info("GET {} -> {}".format(BIOS_FIRMWARE_PATH, response.status_code)) + assert_status_ok(response, BIOS_FIRMWARE_PATH) + + body = response.json() + assert_field_equals(body, "@odata.id", BIOS_FIRMWARE_PATH) + _assert_software_inventory_shape(body, "bios") + assert_field_equals(body, "Description", "Other image") + logger.info("BIOS Version: {!r}".format(body.get("Version"))) + + def test_firmware_switch(self, redfish_client): + """ + Switch (host) firmware inventory entry. + + GET /redfish/v1/UpdateService/FirmwareInventory/switch — validates the + switch SoftwareInventory entry links back to /redfish/v1/Systems/system/Bios + via RelatedItem. + """ + response = redfish_client.get(SWITCH_FIRMWARE_PATH) + logger.info("GET {} -> {}".format(SWITCH_FIRMWARE_PATH, response.status_code)) + assert_status_ok(response, SWITCH_FIRMWARE_PATH) + + body = response.json() + assert_field_equals(body, "@odata.id", SWITCH_FIRMWARE_PATH) + _assert_software_inventory_shape(body, "switch") + assert_field_equals(body, "Description", "Host image") + logger.info("Switch Version: {!r}".format(body.get("Version"))) + + related_ids = [r.get("@odata.id") for r in body.get("RelatedItem", [])] + pytest_assert( + "/redfish/v1/Systems/system/Bios" in related_ids, + "Switch RelatedItem must contain '/redfish/v1/Systems/system/Bios', got: {}".format( + related_ids) + ) diff --git a/tests/redfish/test_redfish_service_root.py b/tests/redfish/test_redfish_service_root.py new file mode 100644 index 00000000000..2dbcb38fc7e --- /dev/null +++ b/tests/redfish/test_redfish_service_root.py @@ -0,0 +1,81 @@ +""" +Test for Redfish Service Root endpoint: GET /redfish/v1 + +Validates that the BMC Redfish service root returns a well-formed response +as defined by the DMTF Redfish specification. +""" +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.redfish.redfish_utils import ( + assert_field_equals, assert_field_contains, assert_field_nonempty, assert_status_ok, +) + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('bmc'), +] + +REQUIRED_FIELDS = ["@odata.type", "@odata.id", "RedfishVersion", "UUID", "Links"] +SERVICE_ROOT = "/redfish/v1" + + +class TestRedfishServiceRoot: + + def test_service_root_accessible(self, redfish_client): + """ + Service root is accessible. + + GET /redfish/v1, validate HTTP 200 and Content-Type is application/json. + """ + response = redfish_client.get(SERVICE_ROOT) + logger.info("HTTP status: {}".format(response.status_code)) + + assert_status_ok(response, SERVICE_ROOT) + content_type = response.headers.get("Content-Type", "") + pytest_assert( + "application/json" in content_type, + "Expected Content-Type to contain 'application/json', got: {!r}".format(content_type) + ) + + def test_service_root_fields(self, redfish_client): + """ + Service root contains required fields. + + Validates DMTF-required fields and SONiC-specific navigation links. + """ + response = redfish_client.get(SERVICE_ROOT) + assert_status_ok(response, SERVICE_ROOT) + + body = response.json() + + for field in REQUIRED_FIELDS: + pytest_assert( + field in body, + "Required field '{}' missing from {} response".format(field, SERVICE_ROOT) + ) + + assert_field_equals(body, "@odata.id", SERVICE_ROOT) + assert_field_contains(body, "@odata.type", "ServiceRoot") + assert_field_nonempty(body, "RedfishVersion") + assert_field_nonempty(body, "UUID") + assert_field_equals(body, "Product", "SONiCBMC") + + # Navigation links + update_service_link = body.get("UpdateService", {}).get("@odata.id", "") + pytest_assert( + update_service_link == "/redfish/v1/UpdateService", + "UpdateService.@odata.id must be '/redfish/v1/UpdateService', got: {!r}".format( + update_service_link) + ) + + systems_link = body.get("Systems", {}).get("@odata.id", "") + pytest_assert( + systems_link == "/redfish/v1/Systems", + "Systems.@odata.id must be '/redfish/v1/Systems', got: {!r}".format(systems_link) + ) + + links = body.get("Links", {}) + pytest_assert("Sessions" in links, "Links.Sessions is missing from {} response".format(SERVICE_ROOT)) diff --git a/tests/route/test_duplicate_route.py b/tests/route/test_duplicate_route.py index 444890343f7..9d793bb3059 100644 --- a/tests/route/test_duplicate_route.py +++ b/tests/route/test_duplicate_route.py @@ -139,19 +139,17 @@ def setup_routes(duthosts, enum_rand_one_per_hwsku_frontend_hostname, if interface_types == 'Loopback': # Get loopback ips intf_ips = get_intf_ips('Loopback', cfg_facts) - pytest_require((len(intf_ips['ipv4']) + len(intf_ips['ipv6'])) > 0, "No IP configured on Loopback0") else: # Get vlan ips intf_ips = get_intf_ips('Vlan', cfg_facts) - pytest_require((len(intf_ips['ipv4']) + len(intf_ips['ipv6'])) > 0, "No IP configured on any Vlan") + + ip_key = 'ipv4' if ip_versions == 4 else 'ipv6' + pytest_require(len(intf_ips[ip_key]) > 0, "No {} configured on {}".format(ip_key, interface_types)) # Generate interfaces and neighbors intf_neighs, str_intf_nexthop = generate_intf_neigh( asichost, 1, ip_versions) - if ip_versions == 4: - prefixes.append(str(random.choice(intf_ips['ipv4'])).split("/")[0]) - else: - prefixes.append(str(random.choice(intf_ips['ipv6'])).split("/")[0]) + prefixes.append(str(random.choice(intf_ips[ip_key])).split("/")[0]) # Setup interface IPs and neighbors prepare_dut(asichost, intf_neighs) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index 2a802a6b290..8bb1669ed08 100644 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -2263,6 +2263,7 @@ def runTest(self): hwsku = self.test_params['hwsku'] platform_asic = self.test_params['platform_asic'] src_dst_asic_diff = self.test_params['src_dst_asic_diff'] + descriptor_size = int(self.test_params.get('descriptor_size', 0)) pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac # get counter names to query @@ -2286,7 +2287,7 @@ def runTest(self): packet_length = 64 if 'cell_size' in self.test_params: cell_size = self.test_params['cell_size'] - cell_occupancy = (packet_length + cell_size - 1) // cell_size + cell_occupancy = (packet_length + cell_size + descriptor_size - 1) // cell_size else: cell_occupancy = 1 @@ -2808,6 +2809,7 @@ def parse_test_params(self): self.dst_port_id = int(self.test_params['dst_port_id']) self.dst_port_ip = self.test_params['dst_port_ip'] self.dst_port_mac = self.dataplane.get_mac(0, self.dst_port_id) + self.descriptor_size = int(self.test_params.get('descriptor_size', 0)) self.ttl = 64 if 'packet_size' in self.test_params: @@ -2818,7 +2820,7 @@ def parse_test_params(self): if 'cell_size' in self.test_params: cell_size = self.test_params['cell_size'] self.cell_occupancy = ( - self.default_packet_length + cell_size - 1) // cell_size + self.default_packet_length + cell_size + self.descriptor_size - 1) // cell_size else: self.cell_occupancy = 1 # Margin used to while crossing the shared headrooom boundary @@ -3010,6 +3012,7 @@ def runTest(self): pkts_num_leak_out = int(self.test_params['pkts_num_leak_out']) pkts_num_trig_pfc = int(self.test_params['pkts_num_trig_pfc']) pkts_num_dismiss_pfc = int(self.test_params['pkts_num_dismiss_pfc']) + descriptor_size = int(self.test_params.get('descriptor_size', 0)) if 'pkts_num_hysteresis' in list(self.test_params.keys()): hysteresis = int(self.test_params['pkts_num_hysteresis']) else: @@ -3053,7 +3056,7 @@ def runTest(self): packet_length = 64 if 'cell_size' in self.test_params: cell_size = self.test_params['cell_size'] - cell_occupancy = (packet_length + cell_size - 1) // cell_size + cell_occupancy = (packet_length + cell_size + descriptor_size - 1) // cell_size else: cell_occupancy = 1 @@ -3554,13 +3557,14 @@ def setUp(self): self.pkts_num_trig_pfc_multi = self.test_params.get('pkts_num_trig_pfc_multi', None) self.pkts_num_hdrm_full = self.test_params['pkts_num_hdrm_full'] self.pkts_num_hdrm_partial = self.test_params['pkts_num_hdrm_partial'] + self.descriptor_size = int(self.test_params.get('descriptor_size', 0)) packet_size = self.test_params.get('packet_size') if packet_size: self.pkt_size = packet_size cell_size = self.test_params.get('cell_size') - self.pkt_size_factor = int(math.ceil(float(packet_size) / cell_size)) + self.pkt_size_factor = int(math.ceil(float(packet_size + self.descriptor_size) / cell_size)) else: self.pkt_size = 64 self.pkt_size_factor = 1 @@ -4973,6 +4977,7 @@ def runTest(self): dut_asic = self.test_params["dut_asic"] update_COUNTER_MARGIN(dut_asic) + descriptor_size = int(self.test_params.get('descriptor_size', 0)) # get counter names to query ingress_counters, egress_counters = get_counter_names(sonic_version) @@ -4986,7 +4991,7 @@ def runTest(self): packet_length = int(self.test_params['packet_size']) cell_size = int(self.test_params['cell_size']) if packet_length != 64: - cell_occupancy = (packet_length + cell_size - 1) // cell_size + cell_occupancy = (packet_length + cell_size + descriptor_size - 1) // cell_size pkts_num_trig_egr_drp //= cell_occupancy # It is possible that pkts_num_trig_egr_drp * cell_occupancy < original pkts_num_trig_egr_drp, # which probably can fail the assert (xmit_counters[EGRESS_DROP] > xmit_counters_base[EGRESS_DROP]) @@ -5535,13 +5540,14 @@ def runTest(self): platform_asic = self.test_params['platform_asic'] margin_lower_bound = self.test_params.get('pkts_num_margin_lower_bound', 0) ip_type = self.test_params.get('ip_type', 'ipv4') + descriptor_size = int(self.test_params.get('descriptor_size', 0)) if 'packet_size' in list(self.test_params.keys()): packet_length = int(self.test_params['packet_size']) else: packet_length = 64 - cell_occupancy = (packet_length + cell_size - 1) // cell_size + cell_occupancy = (packet_length + cell_size + descriptor_size - 1) // cell_size # Prepare TCP packet data ttl = 64 @@ -5827,6 +5833,7 @@ def runTest(self): cell_size = int(self.test_params['cell_size']) hwsku = self.test_params['hwsku'] platform_asic = self.test_params['platform_asic'] + descriptor_size = int(self.test_params.get('descriptor_size', 0)) # Prepare TCP packet data ttl = 64 @@ -5835,7 +5842,7 @@ def runTest(self): else: default_packet_length = 64 - cell_occupancy = (default_packet_length + cell_size - 1) // cell_size + cell_occupancy = (default_packet_length + cell_size + descriptor_size - 1) // cell_size pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac is_dualtor = self.test_params.get('is_dualtor', False) def_vlan_mac = self.test_params.get('def_vlan_mac', None) @@ -6190,13 +6197,14 @@ def runTest(self): platform_asic = self.test_params['platform_asic'] dut_asic = self.test_params['dut_asic'] ip_type = self.test_params.get('ip_type', 'ipv4') + descriptor_size = int(self.test_params.get('descriptor_size', 0)) if 'packet_size' in list(self.test_params.keys()): packet_length = int(self.test_params['packet_size']) else: packet_length = 64 - cell_occupancy = (packet_length + cell_size - 1) // cell_size + cell_occupancy = (packet_length + cell_size + descriptor_size - 1) // cell_size # Prepare TCP packet data ttl = 64 diff --git a/tests/sflow/test_sflow.py b/tests/sflow/test_sflow.py index bd4e104ab45..279888a828a 100644 --- a/tests/sflow/test_sflow.py +++ b/tests/sflow/test_sflow.py @@ -92,7 +92,7 @@ def setup(duthosts, rand_one_dut_hostname, ptfhost, tbinfo, config_sflow_feature # -------- Testing ---------- yield # -------- Teardown ---------- - config_reload(duthost, config_source='minigraph', wait=120) + config_reload(duthost, config_source='minigraph', wait=120, override_config=True) # ---------------------------------------------------------------------------------- @@ -666,9 +666,15 @@ def testNonDefaultAgent(self, duthosts, rand_one_dut_hostname, partial_ptf_runne def testDelAgent(self, duthosts, rand_one_dut_hostname, partial_ptf_runner): duthost = duthosts[rand_one_dut_hostname] + read_agent_cmd = "docker exec sflow grep -w 'agentIP' /etc/hsflowd.auto 2>/dev/null | cut -d '=' -f 2" + previous_agent_ip = duthost.shell(read_agent_cmd, module_ignore_errors=True)['stdout'].strip() duthost.shell(" config sflow agent-id del") verify_show_sflow(duthost, status='up', agent_id='default') wait_until(30, 5, 0, verify_sflow_config_apply, duthost) + # Wait for hsflowd to rewrite /etc/hsflowd.auto with the new agentIP, + # otherwise get_default_agent below reads the stale previous value. + wait_until(60, 2, 0, lambda: duthost.shell( + read_agent_cmd, module_ignore_errors=True)['stdout'].strip() not in ('', previous_agent_ip)) agent_ip = get_default_agent(duthost) # Verify whether the samples are received with previously configured agent ip partial_ptf_runner( diff --git a/tests/show_techsupport/test_techsupport.py b/tests/show_techsupport/test_techsupport.py index 80ddefd906d..a44c0fd23bf 100644 --- a/tests/show_techsupport/test_techsupport.py +++ b/tests/show_techsupport/test_techsupport.py @@ -367,7 +367,8 @@ def collect_platform_dump_files(folder_name): def gen_dump_file(duthost, since): logger.debug("Running show techsupport ... ") - wait_until(300, 20, 0, execute_command, duthost, str(since)) + pytest_assert(wait_until(300, 20, 0, execute_command, duthost, str(since)), + "show techsupport command failed to succeed within timeout") tar_file = [j for j in pytest.tar_stdout.split('\n') if j != ''][-1] return tar_file diff --git a/tests/smartswitch/common/device_utils_dpu.py b/tests/smartswitch/common/device_utils_dpu.py index e783084aec1..3c2fa6251bd 100644 --- a/tests/smartswitch/common/device_utils_dpu.py +++ b/tests/smartswitch/common/device_utils_dpu.py @@ -253,6 +253,17 @@ def check_dpu_module_status(duthost, power_status, dpu_name): return False +def _wait_for_dpu_status(duthost, dpu_name, expected_status, action): + """ + Poll until the DPU module reaches *expected_status* ('on' or 'off'). + """ + pytest_assert( + wait_until(DPU_MAX_ONLINE_TIMEOUT, DPU_TIME_INT, 0, + check_dpu_module_status, duthost, expected_status, dpu_name), + f"DPU {dpu_name} is not {expected_status} after '{action}'" + ) + + def check_dpus_module_status(duthost, dpu_list, power_status, wait_timeout=30): """ Check module status of given DPU list @@ -789,34 +800,28 @@ def dpus_shutdown_and_check(duthost, dpu_list, num_dpu_modules): duthost.shell, f"sudo config chassis modules shutdown {dpu_name}" ) - executor.submit( - wait_until, DPU_MAX_ONLINE_TIMEOUT, DPU_TIME_INT, 0, - check_dpu_module_status, duthost, "off", dpu_name - ) + executor.submit(_wait_for_dpu_status, duthost, dpu_name, "off", "shutdown") def dpus_startup_and_check(duthost, dpu_list, num_dpu_modules): """ - Parallely Execute DPU startup for given DPU list + Serialize DPU startup for given DPU list Waits and checks parallely whether DPU is actually UP Args: duthost: Host handle dpu_list: List of DPUs to be startup - + num_dpu_modules: number of dpu modules Returns: Returns Nothing """ + logging.info("Dispatching startup commands for DPUs serially") + for dpu_name in dpu_list: + duthost.shell(f"sudo config chassis modules startup {dpu_name}") + with SafeThreadPoolExecutor(max_workers=num_dpu_modules) as executor: logging.info("Check startup of DPUs in parallel") for dpu_name in dpu_list: - executor.submit( - duthost.shell, - f"sudo config chassis modules startup {dpu_name}" - ) - executor.submit( - wait_until, DPU_MAX_ONLINE_TIMEOUT, DPU_TIME_INT, 0, - check_dpu_module_status, duthost, "on", dpu_name - ) + executor.submit(_wait_for_dpu_status, duthost, dpu_name, "on", "startup") def check_midplane_status(duthost, dpu_ip, expected_status): diff --git a/tests/smartswitch/platform_tests/conftest.py b/tests/smartswitch/platform_tests/conftest.py new file mode 100644 index 00000000000..b96c534c340 --- /dev/null +++ b/tests/smartswitch/platform_tests/conftest.py @@ -0,0 +1,63 @@ +""" +Pytest fixtures for SmartSwitch platform tests. +""" +import logging +import pytest +from pytest_ansible.errors import AnsibleConnectionFailure +from tests.common.platform.processes_utils import wait_critical_processes +from tests.common.reboot import wait_for_startup +from tests.smartswitch.common.device_utils_dpu import ( # noqa: F401 + dpus_startup_and_check, SWITCH_MAX_DELAY, SWITCH_MAX_TIMEOUT, num_dpu_modules +) +from tests.common.platform.device_utils import platform_api_conn, start_platform_api_service # noqa: F401 + + +@pytest.fixture(autouse=True) +def ensure_all_dpus_ready(duthosts, + enum_rand_one_per_hwsku_hostname, + localhost, + num_dpu_modules): # noqa: F811 + """ + Teardown fixture: after each test case, ensure all DPUs are back online. + If any DPU is found offline at the end of a test, it will be started up + before the next test begins. + """ + yield + + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + dpu_names = ["DPU{}".format(i) for i in range(num_dpu_modules)] + + def _get_offline_dpus(): + """Single shell call to find all offline DPUs.""" + output = duthost.shell("show chassis module status")["stdout"] + return [ + dpu for dpu in dpu_names + if any(dpu in line and "offline" in line.lower() + for line in output.splitlines()) + ] + + def _do_dpu_recovery(): + offline = _get_offline_dpus() + if offline: + logging.info("DPUs found offline after test: %s. Bringing them back UP...", offline) + dpus_startup_and_check(duthost, offline, num_dpu_modules) + logging.info("All DPUs are back online after recovery.") + else: + logging.info("All DPUs are online after test. No recovery needed.") + + try: + _do_dpu_recovery() + except AnsibleConnectionFailure: + logging.warning( + "DUT %s unreachable in teardown (still rebooting?); waiting for it to come back up", + duthost.hostname + ) + try: + wait_for_startup(duthost, localhost, SWITCH_MAX_DELAY, SWITCH_MAX_TIMEOUT) + wait_critical_processes(duthost) + logging.info("DUT %s is back up; retrying DPU recovery", duthost.hostname) + _do_dpu_recovery() + except Exception as e: + logging.warning("DPU recovery after DUT reboot wait failed (non-fatal): %s", e) + except Exception as e: + logging.warning("DPU recovery in teardown failed (non-fatal): %s", e) diff --git a/tests/smartswitch/platform_tests/test_reload_dpu.py b/tests/smartswitch/platform_tests/test_reload_dpu.py index 6697826095d..c51fe88db79 100644 --- a/tests/smartswitch/platform_tests/test_reload_dpu.py +++ b/tests/smartswitch/platform_tests/test_reload_dpu.py @@ -41,34 +41,7 @@ def invocation_type(request): return request.param -@pytest.fixture(autouse=True) -def ensure_dpus_up_after_test(duthosts, - enum_rand_one_per_hwsku_hostname, - num_dpu_modules): # noqa: F811 - """ - Teardown fixture: after each test case, ensure all DPUs are back online. - If any DPU is found offline at the end of a test, it will be started up - before the next test begins. - """ - yield - - duthost = duthosts[enum_rand_one_per_hwsku_hostname] - dpu_names = ["DPU{}".format(i) for i in range(num_dpu_modules)] - try: - offline_dpus = [ - dpu for dpu in dpu_names - if not check_dpu_module_status(duthost, "on", dpu) - ] - if offline_dpus: - logging.info("DPUs found offline after test: %s. Bringing them back UP...", offline_dpus) - dpus_startup_and_check(duthost, offline_dpus, num_dpu_modules) - logging.info("All DPUs are back online after recovery.") - else: - logging.info("All DPUs are online after test. No recovery needed.") - except Exception as e: - logging.warning("DPU recovery in teardown failed (non-fatal): %s", e) - - +@pytest.mark.disable_loganalyzer def test_dpu_status_post_switch_reboot(duthosts, dpuhosts, enum_rand_one_per_hwsku_hostname, localhost, @@ -415,6 +388,7 @@ def test_cold_reboot_dpus(duthosts, dpuhosts, enum_rand_one_per_hwsku_hostname, pre_boot_times=pre_boot_times) +@pytest.mark.disable_loganalyzer def test_cold_reboot_switch(duthosts, dpuhosts, enum_rand_one_per_hwsku_hostname, platform_api_conn, num_dpu_modules, localhost): # noqa: F811, E501 """ @@ -439,8 +413,8 @@ def test_cold_reboot_switch(duthosts, dpuhosts, enum_rand_one_per_hwsku_hostname logging.info("Starting switch reboot...") logging.info("Recording DPU boot times before switch cold reboot") pre_boot_times = get_all_dpu_uptimes(dpuhosts, dpu_on_list) - - perform_reboot(duthost, REBOOT_TYPE_COLD, None) + reboot(duthost, localhost, reboot_type=REBOOT_TYPE_COLD, + wait_for_ssh=False) logging.info("Executing post test check") post_test_switch_check(duthost, localhost, diff --git a/tests/snappi_tests/dash/ha/ha_helper.py b/tests/snappi_tests/dash/ha/ha_helper.py index 2f4943e11ee..a0e24c72516 100644 --- a/tests/snappi_tests/dash/ha/ha_helper.py +++ b/tests/snappi_tests/dash/ha/ha_helper.py @@ -39,7 +39,7 @@ def run_ha_test(duthosts, localhost, tbinfo, ha_test_case, config_npu_dpu, confi # Traffic Starts if ha_test_case == 'cps': - api = run_cps_search(api, file_name, initial_cps_value, passing_dpus) + api = run_cps_search(api, config_snappi_l47, file_name, initial_cps_value, passing_dpus) logger.info("Test Ending") elif ha_test_case == 'planned_switchover': api = run_planned_switchover(duthosts, tbinfo, file_name, api, initial_cps_value) @@ -500,19 +500,22 @@ def window_indices(n): return retries_resets -def run_cps_search(api, file_name, initial_cps_value, passing_dpus): +def run_cps_search(api, config_snappi_l47, file_name, initial_cps_value, passing_dpus): error_threshold = 0.01 # noqa: F841 - MAX_CPS = 30000000 + MAX_CPS_PER_CARD = 4500000 + MAX_CPS = len(config_snappi_l47['ports_list']['Traffic1@Network1']) * MAX_CPS_PER_CARD MIN_CPS = 0 threshold = 1000000 test_iteration = 1 test_value = initial_cps_value + num_test_cards = len(config_snappi_l47['ports_list']['Traffic1@Network1']) activityList_url = "ixload/test/activeTest/communityList/0" constraint_url = "ixload/test/activeTest/communityList/0/activityList/0" releaseConfig_url = "ixload/test/operations/abortAndReleaseConfigWaitFinish" testRuns = [] + # Find MAX CPS, track errors while ((MAX_CPS - MIN_CPS) > threshold): collector = ContinuousMetricsCollector(collection_interval=1) @@ -618,21 +621,27 @@ def run_cps_search(api, file_name, initial_cps_value, passing_dpus): logger.info('Test Iteration Pass') test_result = "Pass" MIN_CPS = test_value - test_value = (MAX_CPS + MIN_CPS) / 2 + test_value = int(MAX_CPS + MIN_CPS / 2) + if test_value > int(num_test_cards * MAX_CPS_PER_CARD): + # limit the max CPS setting to 4.5M cps per card + test_value = int(num_test_cards * MAX_CPS_PER_CARD) else: logger.info('Test Iteration Fail') test_result = "Fail" MAX_CPS = test_value - test_value = (MAX_CPS + MIN_CPS) / 2 + test_value = int((MAX_CPS + MIN_CPS) / 2) if len(passing_dpus) == 0: passing_dpus = 0 - columns = ['#Run', 'CPS Objective', 'Max CPS', f'{err_maxname}', 'Number of DPUs (Indexes)', 'Test Result'] + columns = ['#Run', 'CPS Test Objective', 'Max CPS', f'{err_maxname}', 'Number of DPUs (Indexes)', 'Test Result'] testRuns.append([test_iteration, cps_objective_value, cps_max, err_maxvalue, passing_dpus, test_result]) table = tabulate(testRuns, headers=columns, tablefmt='psql') logger.info(table) + max_cps_testruns = max(run[2] for run in testRuns) + logger.info(f"The max CPS from all test runs is {max_cps_testruns}") + logger.info("Iteration Ended...") logger.info('MIN_CPS = %d' % MIN_CPS) logger.info('Current MAX_CPS = %d' % MAX_CPS) @@ -653,6 +662,10 @@ def run_cps_search(api, file_name, initial_cps_value, passing_dpus): cs.app.state = 'stop' # cs.app.state.START api.set_control_state(cs) + logger.info("Test Iteration Complete:") + logger.info(table) + logger.info(f"Final max CPS from all test runs is {max_cps_testruns}") + return api diff --git a/tests/snappi_tests/dash/sample_testbed.yaml b/tests/snappi_tests/dash/sample_testbed.yaml index dc5fac7911b..661fc64f499 100644 --- a/tests/snappi_tests/dash/sample_testbed.yaml +++ b/tests/snappi_tests/dash/sample_testbed.yaml @@ -1,30 +1,43 @@ ---- - - conf-name: vms-snappi-sonic group-name: vms6-1 topo: ptf32 ptf_image_name: docker-ptf-snappi - ptf: snappi_sonic_l47 + ptf: snappi_sonic_ixl ptf_ip: 10.1.1.3 - service_type: vnet2vnet + service_type: privatelink + #service_type: vnet2vnet + #vxlan_port: 64200 + #vxlan_src_port: 65330 l47_gateway: 10.1.1.3 l47_tg_servermac: 140776176680961 l47_tg_clientmac: 140776176680962 dut_mac: 40501075136016 - l47_version: 11.00.0.292 + ixos_version: 26.0.2600.14 + l47_version: 26.0.0.94 chassis_ip: 10.1.1.4 + eni_per_dpu: 32 + chassis_user_login: root + chassis_user_passwd: somePassWordForTesting + dpu_target_username: admin + dpu_target_passwd: YourPaSsWoRd + clean_l47trafficgen_staticarps: True uhd_ip: 10.1.1.5 + vxlan_endpoint_vni: 1000 dpu_active_ip: 172.35.1.1 dpu_standby_ip: 172.35.1.2 - dpu_active_mac: b0:8d:57:cd:36:ef - dpu_standby_mac: ec:19:2e:f1:ca:af + dpu_active_mac: bb:dd:55:cc:33:ee + dpu_standby_mac: ee:99:22:ff:cc:ff dpu_active_if: Ethernet224 dpu_standby_if: Ethernet240 num_cps_cards: 8 - num_tcpbg_cards: 4 - num_udpbg_cards: 1 + num_tcpbg_cards: 0 + num_udpbg_cards: 0 num_dpus: 1 - dpu_ports_list: 5, 6 + dpu_ports_list: [5, 6] + active_ethpass_ip: 220.0.4.1 + active_mac: 44:dd:ee:33:44:00 + standby_ethpass_ip: 220.0.4.2 + standby_mac: 22:55:44:22:55:11 ports_list: Traffic1@Network1: - [ 1, 1, 1 ] diff --git a/tests/snappi_tests/dash/test_cps.py b/tests/snappi_tests/dash/test_cps.py index 17285a03ad8..9cd32130f89 100755 --- a/tests/snappi_tests/dash/test_cps.py +++ b/tests/snappi_tests/dash/test_cps.py @@ -3,6 +3,8 @@ from tests.common.snappi_tests.snappi_fixtures import config_uhd_connect # noqa F401 from tests.common.snappi_tests.ixload.snappi_fixtures import config_snappi_l47 # noqa F401 from tests.common.snappi_tests.ixload.snappi_fixtures import config_npu_dpu # noqa F401 +from tests.common.snappi_tests.ixload.snappi_fixtures import setup_config_snappi_l47, setup_config_npu_dpu # noqa F401 +from tests.common.snappi_tests.snappi_fixtures import setup_config_uhd_connect # noqa F401 from concurrent.futures import ThreadPoolExecutor, as_completed import pytest @@ -31,7 +33,7 @@ def test_cps_baby_hero( request, ): # noqa F811 - fixture_names = ["config_snappi_l47", "config_npu_dpu", "config_uhd_connect"] + fixture_names = ["config_snappi_l47", "config_npu_dpu", "config_uhd_connect"] # noqa: F841 results = {} errors = {} @@ -40,14 +42,38 @@ def test_cps_baby_hero( if sw1 is False: pytest.skip("Skipping test since is not a smartswitch") - def _resolve_fixture(name): - # Resolve the fixture value on-demand - return request.getfixturevalue(name) - + def _run_config_snappi_l47(): + try: + return setup_config_snappi_l47(request, duthost, tbinfo, ha_test_case) + except Exception as e: + raise e + + def _run_config_npu_dpu(): + try: + return setup_config_npu_dpu(request, duthost, localhost, tbinfo, ha_test_case) + except Exception as e: + raise e + + def _run_config_uhd_connect(): + try: + return setup_config_uhd_connect(request, tbinfo, ha_test_case) + except Exception as e: + raise e + + # Run the setup functions in parallel with ThreadPoolExecutor(max_workers=3) as ex: - fm = {ex.submit(_resolve_fixture, name): name for name in fixture_names} - for fut in as_completed(fm): - name = fm[fut] + future_snappi = ex.submit(_run_config_snappi_l47) + future_npu = ex.submit(_run_config_npu_dpu) + future_uhd = ex.submit(_run_config_uhd_connect) + + futures = { + future_snappi: "config_snappi_l47", + future_npu: "config_npu_dpu", + future_uhd: "config_uhd_connect" + } + + for fut in as_completed(futures): + name = futures[fut] try: results[name] = fut.result() except Exception as e: diff --git a/tests/snappi_tests/dataplane/conftest.py b/tests/snappi_tests/dataplane/conftest.py new file mode 100644 index 00000000000..47461119058 --- /dev/null +++ b/tests/snappi_tests/dataplane/conftest.py @@ -0,0 +1,102 @@ +"""Pytest fixtures for snappi dataplane tests.""" +import pytest + + +@pytest.fixture(scope="module") +def set_primary_chassis(snappi_api, fanout_graph_facts_multidut, duthosts): + """Set primary chassis for multi-chassis Ixia setups.""" + from tests.common.helpers.assertions import pytest_assert + + primary_chassis = duthosts[0].host.options['variable_manager'] \ + ._hostvars[duthosts[0].hostname].get('chassis_chain_primary', None) + if len(fanout_graph_facts_multidut) == 1 and primary_chassis: + slave_chassis = [ + fanout_graph_facts_multidut[fanout]['device_info']['mgmtip'] + for fanout in fanout_graph_facts_multidut + ] + elif len(fanout_graph_facts_multidut) > 1: + slave_chassis = [] + for fanout in fanout_graph_facts_multidut: + if 'primary' in fanout_graph_facts_multidut[fanout]['device_info']['Type'].lower(): + primary_chassis = fanout_graph_facts_multidut[fanout]['device_info']['mgmtip'] + else: + slave_chassis.append(fanout_graph_facts_multidut[fanout]['device_info']['mgmtip']) + if not primary_chassis: + pytest_assert(False, "No primary chassis found in the ansible/ devices.csv file") + else: + return False + ixnconfig = snappi_api.ixnet_specific_config + chassis_chain1 = ixnconfig.chassis_chains.add() + chassis_chain1.primary = primary_chassis + chassis_chain1.topology = chassis_chain1.STAR + slaves = [ # noqa F841 + chassis_chain1.secondary.add( + location=slave, + sequence_id=index, + cable_length='6' + ) + for index, slave in enumerate(slave_chassis, start=2) + ] + return snappi_api + + +@pytest.fixture(scope="module") +def create_snappi_config(snappi_api, get_snappi_ports): + """Create snappi configuration builder.""" + from tests.snappi_tests.dataplane.files.helper import create_snappi_l1config + from tests.common.helpers.assertions import pytest_assert + + def _create_snappi_config(snappi_extra_params): + pytest_assert(snappi_extra_params.protocol_config, "No protocol configuration provided in snappi_extra_params") + + # Extract actual ports from protocol_config instead of using all get_snappi_ports + actual_ports = [] + for role, pconfig in snappi_extra_params.protocol_config.items(): + actual_ports.extend(pconfig['ports']) + + config = create_snappi_l1config(snappi_api, actual_ports, snappi_extra_params) + snappi_obj_handles = {k: {"ip": [], "network_group": []} for k in snappi_extra_params.protocol_config} + + for role, pconfig in snappi_extra_params.protocol_config.items(): + is_ipv4 = True if pconfig['subnet_type'] == 'IPv4' else False + for index, port_data in enumerate(pconfig['ports']): + device = config.devices.device(name=f"{role} Topology {index}")[-1] + eth = device.ethernets.add(name=f"{role} Ethernet_{index}", mac=port_data["src_mac_address"]) + eth.connection.port_name = f"Port_{port_data['port_id']}" + ip_name = f"{role} {'IPv4' if is_ipv4 else 'IPv6'}_{index}" + ip_layer = getattr(eth, 'ipv4_addresses' if is_ipv4 else 'ipv6_addresses').add( + name=ip_name, + address=port_data["ipAddress"], + gateway=port_data["ipGateway"], + prefix=int(port_data["prefix"]) + ) + snappi_obj_handles[role]["ip"].append(ip_layer.name) + + if pconfig.get("protocol_type", False) and pconfig['protocol_type'] == "bgp": + bgp = device.bgp + bgp.router_id = port_data["ipGateway"] if is_ipv4 else '1.1.1.1' + iface = bgp.ipv4_interfaces.add() if is_ipv4 else bgp.ipv6_interfaces.add() + setattr(iface, 'ipv4_name' if is_ipv4 else 'ipv6_name', ip_layer.name) + peer = iface.peers.add( + name=f"{role} BGP{'' if is_ipv4 else '+'}_{index}", + as_type='ebgp', + peer_address=port_data["ipGateway"], + as_number=port_data["asn"] + ) + + if 'route_ranges' in pconfig and pconfig['route_ranges']: + for route_range in pconfig['route_ranges']: + v4_routes = peer.v4_routes.add( + name=f"{role} Network Group {index}", + addresses=route_range + ) + v4_routes.addresses.add( + address=route_range[0], + prefix=route_range[1], + count=route_range[2] + ) + snappi_obj_handles[role]["network_group"].append(v4_routes.name) + + return config, snappi_obj_handles + + return _create_snappi_config diff --git a/tests/snappi_tests/dataplane/files/helper.py b/tests/snappi_tests/dataplane/files/helper.py index d8bcef2080e..d0141be3111 100644 --- a/tests/snappi_tests/dataplane/files/helper.py +++ b/tests/snappi_tests/dataplane/files/helper.py @@ -194,18 +194,37 @@ def get_duthost_bgp_details(duthosts, get_snappi_ports, subnet_type): # noqa (subnet_type == 'ipv6' and ip_obj.version == 6)): peer_to_gateway[port] = (str(ip_obj.ip), ip_obj.network.prefixlen) mac_address_generator = get_macs("101700000011", len(get_snappi_ports)) + valid_ports = [] for index, port in enumerate(get_snappi_ports): if port['duthost'] == duthost: # Get the IP address of the peer port port['router_mac_address'] = port['duthost'].facts['router_mac'] port['src_mac_address'] = mac_address_generator[index] - port['ipGateway'] = peer_to_gateway.get(port['peer_port'])[0] - port['ipAddress'] = gateway_to_bgp.get(port['ipGateway'])['ipAddress'] - port['asn'] = gateway_to_bgp.get(port['ipGateway'])['asn'] - port['prefix'] = peer_to_gateway.get(port['peer_port'])[1] - port['subnet'] = str(peer_to_gateway.get(port['peer_port'])[0]) \ - + "/" + str(peer_to_gateway.get(port['peer_port'])[1]) # noqa: E127 - return get_snappi_ports + peer_info = peer_to_gateway.get(port['peer_port']) + if peer_info is None: + logger.warning( + "Port %s not found in peer_to_gateway mapping, skipping", + port['peer_port'] + ) + continue + port['ipGateway'] = peer_info[0] + bgp_info = gateway_to_bgp.get(port['ipGateway']) + if bgp_info is None: + logger.warning( + "No BGP neighbor found for gateway %s on port %s, skipping", + port['ipGateway'], port['peer_port'] + ) + continue + port['ipAddress'] = bgp_info['ipAddress'] + port['asn'] = bgp_info['asn'] + port['prefix'] = peer_info[1] + port['subnet'] = str(peer_info[0]) + "/" + str(peer_info[1]) + valid_ports.append(port) + else: + # Port belongs to different dut, keep it if it was already processed + if 'ipAddress' in port: + valid_ports.append(port) + return valid_ports def get_duthost_vlan_details(duthosts, get_snappi_ports, subnet_type): # noqa F811 diff --git a/tests/snappi_tests/dataplane/files/max_throughput_config.yaml b/tests/snappi_tests/dataplane/files/max_throughput_config.yaml new file mode 100644 index 00000000000..b23cf11a773 --- /dev/null +++ b/tests/snappi_tests/dataplane/files/max_throughput_config.yaml @@ -0,0 +1,134 @@ +# Max Throughput Minimum Packet Size Test Configuration + +min_ports: 30 +traffic_duration_sec: 60 +line_rate_pct: 100 +tolerance_pkt_size_offset: 10 + +srv6: + locator_name: "loc1" + locator_prefix: "fcbb:bbbb:1::" + block_len: 32 + node_len: 16 + func_len: 0 + arg_len: 0 + sid_action: "uN" + sid_ip: "fcbb:bbbb:1::" + decap_vrf: "default" + decap_dscp_mode: "pipe" + outer_dst_ip: "fcbb:bbbb:1:11a::2" + +# Each scenario defines which DUT features are enabled (true) or disabled (false). +scenarios: + all_features: + description: "DataAcl + Everflow + EverflowV6 + IPinIP Decap + uSID Decap" + dataacl: true + everflow: true + everflowv6: true + ipinip_decap: true + usid_decap: true + + dataacl_usid: + description: "DataAcl + uSID Decap" + dataacl: true + everflow: false + everflowv6: false + ipinip_decap: false + usid_decap: true + + dataacl_ipinip_usid: + description: "DataAcl + IPinIP Decap + uSID Decap" + dataacl: true + everflow: false + everflowv6: false + ipinip_decap: true + usid_decap: true + + ipinip_usid: + description: "IPinIP Decap + uSID Decap" + dataacl: false + everflow: false + everflowv6: false + ipinip_decap: true + usid_decap: true + + dataacl_only: + description: "DataAcl only" + dataacl: true + everflow: false + everflowv6: false + ipinip_decap: false + usid_decap: false + + no_features: + description: "No ACLs + No Decap" + dataacl: false + everflow: false + everflowv6: false + ipinip_decap: false + usid_decap: false + +# Optional aliases for lab names or platform names that differ from HwSku. +platform_aliases: + "sn5640": "Mellanox-SN5640-C512S2" + "Nvidia 5640": "Mellanox-SN5640-C512S2" + "Arista 7260": "Arista-7260CX3-C64" + "Cisco 8102": "Cisco-8102-C64" + +# Per-platform minimum packet sizes (bytes) for zero-drop at 100% line rate. +platform_thresholds: + "Mellanox-SN5640-C512S2": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Mellanox-SN5640-C448O16": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Arista-7260CX3-C64": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Arista-7260CX3-D108C8": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Cisco-8102-C64": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Cisco-8102-28FH-DPU-O": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 + + "Arista-7060X6-64PE-B-C512S2": + all_features: 500 + dataacl_usid: 450 + dataacl_ipinip_usid: 500 + ipinip_usid: 450 + dataacl_only: 400 + no_features: 400 diff --git a/tests/snappi_tests/dataplane/files/max_throughput_helper.py b/tests/snappi_tests/dataplane/files/max_throughput_helper.py new file mode 100644 index 00000000000..7b007651ecf --- /dev/null +++ b/tests/snappi_tests/dataplane/files/max_throughput_helper.py @@ -0,0 +1,563 @@ +""" +Helper module for the SRv6 max throughput minimum packet size test. +""" +import os +import json +import yaml +import logging +import time + +from tests.common.config_reload import config_reload +from tests.common.helpers.srv6_helper import ( + create_srv6_locator, + create_srv6_sid, + del_srv6_locator, + del_srv6_sid, + SRv6, +) + +logger = logging.getLogger(__name__) + +CONFIG_FILE = os.path.join(os.path.dirname(__file__), "max_throughput_config.yaml") + + +def _load_json_output(output): + """Load JSON output that may include informational prefix lines.""" + start = output.find("{") + if start == -1: + raise ValueError("No JSON object found in output: {}".format(output)) + return json.loads(output[start:]) + + +def _normalize_platform_name(platform): + """Normalize platform names for deterministic fuzzy matching.""" + return platform.lower().replace("-", "").replace("_", "").replace(" ", "") + + +def load_max_throughput_config(): + """Load the YAML config for the max throughput test.""" + with open(CONFIG_FILE, "r") as f: + return yaml.safe_load(f) + + +def get_platform_thresholds(duthost, config): + """Look up min-packet-size thresholds for the DUT's platform. Returns None if unknown.""" + platform = duthost.facts.get("hwsku", "") + thresholds = config.get("platform_thresholds", {}) + aliases = config.get("platform_aliases", {}) + + if platform in thresholds: + return thresholds[platform] + + if platform in aliases: + return thresholds.get(aliases[platform]) + + normalized_platform = _normalize_platform_name(platform) + normalized_aliases = { + _normalize_platform_name(alias): target + for alias, target in aliases.items() + } + if normalized_platform in normalized_aliases: + return thresholds.get(normalized_aliases[normalized_platform]) + + # Prefer the most specific platform key over the first partial match. + matches = [] + for key in thresholds: + normalized_key = _normalize_platform_name(key) + if normalized_key and normalized_key in normalized_platform: + matches.append((len(normalized_key), key)) + + if matches: + _, key = max(matches) + return thresholds[key] + + return None + + +# --------------------------------------------------------------------------- +# Feature toggle helpers +# --------------------------------------------------------------------------- + +def _get_front_panel_ports(duthost): + """Return list of admin-up front-panel Ethernet ports on the DUT.""" + result = duthost.shell("portstat -j")["stdout"] + port_data = _load_json_output(result) + # STATE: U=Up, D=Down, X=Disabled (admin-down) + return sorted( + [port for port, stats in port_data.items() if stats.get("STATE") != "X"], + key=lambda p: (len(p), p), + ) + + +def _get_portchannel_members(duthost): + """Return Ethernet ports that are members of PortChannels.""" + result = duthost.shell( + "sonic-db-cli CONFIG_DB keys 'PORTCHANNEL_MEMBER|*'", + module_ignore_errors=True, + ) + members = set() + if result["rc"] != 0: + return members + + for line in result["stdout_lines"]: + parts = line.split("|") + if len(parts) >= 3: + members.add(parts[2]) + return members + + +def _get_portchannels(duthost): + """Return configured PortChannel interface names.""" + result = duthost.shell( + "sonic-db-cli CONFIG_DB keys 'PORTCHANNEL|*'", + module_ignore_errors=True, + ) + portchannels = [] + if result["rc"] != 0: + return portchannels + + for line in result["stdout_lines"]: + parts = line.split("|") + if len(parts) >= 2: + portchannels.append(parts[1]) + return sorted(portchannels, key=lambda p: (len(p), p)) + + +def _get_acl_bind_ports(duthost): + """Return ACL-bindable front-panel interfaces.""" + portchannel_members = _get_portchannel_members(duthost) + standalone_ports = [ + port for port in _get_front_panel_ports(duthost) + if port not in portchannel_members + ] + bind_ports = _get_portchannels(duthost) + standalone_ports + if not bind_ports: + raise RuntimeError( + "No ACL-bindable front-panel interfaces found on {}".format( + duthost.hostname + ) + ) + return bind_ports + + +def _dataacl_exists(duthost): + """Check if DATAACL table exists.""" + lines = duthost.shell(cmd="show acl table DATAACL")["stdout_lines"] + return any("DATAACL" in line for line in lines) + + +def _verify_acl_table_ports(duthost, table_name, expected_ports): + """Verify that the ACL table is bound to the expected ports.""" + output = duthost.shell("show acl table {}".format(table_name))["stdout"] + for port in expected_ports: + if port not in output: + logger.warning("ACL table %s not attached to port %s", table_name, port) + return False + logger.info("ACL table %s verified on all %d ports", table_name, len(expected_ports)) + return True + + +def _verify_acl_table_removed(duthost, table_name): + """Verify that the ACL table no longer exists.""" + output = duthost.shell("show acl table {}".format(table_name))["stdout"] + if table_name in output: + logger.warning("ACL table %s still exists after removal", table_name) + return False + logger.info("ACL table %s confirmed removed", table_name) + return True + + +# --- DataACL --- + +def enable_dataacl(duthost): + """Add DATAACL L3 table bound to ACL-bindable front-panel interfaces.""" + if _dataacl_exists(duthost): + logger.info("DATAACL already exists on %s, skipping", duthost.hostname) + return + ports = _get_acl_bind_ports(duthost) + cmd = "config acl add table DATAACL L3 -p {}".format(",".join(ports)) + logger.info("Enabling DATAACL on %s", duthost.hostname) + duthost.shell(cmd) + duthost.shell("config save -y") + if not _verify_acl_table_ports(duthost, "DATAACL", ports): + raise RuntimeError( + "DATAACL not attached to expected interfaces on {}".format( + duthost.hostname + ) + ) + + +def disable_dataacl(duthost): + """Remove DATAACL table if it exists.""" + if not _dataacl_exists(duthost): + logger.info("DATAACL does not exist on %s, skipping removal", duthost.hostname) + return + logger.info("Removing DATAACL on %s", duthost.hostname) + duthost.shell("config acl remove table DATAACL") + duthost.shell("config save -y") + if not _verify_acl_table_removed(duthost, "DATAACL"): + raise RuntimeError("DATAACL not fully removed on {}".format(duthost.hostname)) + + +# --- Everflow (IPv4 mirror) --- + +EVERFLOW_SESSION = "max_tput_ev4_session" +EVERFLOW_TABLE = "EVERFLOW" +MIRROR_SRC_IP = "10.10.10.1" +MIRROR_DST_IP = "10.10.10.2" +MIRROR_DSCP = "8" +MIRROR_TTL = "64" +MIRROR_QUEUE = "0" +MIRROR_GRE_TYPES = { + "mellanox": "35145", # 0x8949 + "barefoot": "8939", # 0x22EB + "cisco-8000": "35006", # 0x88BE +} +DEFAULT_MIRROR_GRE_TYPE = "35006" # 0x88BE + + +def _mirror_session_exists(duthost, session_name): + output = duthost.shell("show mirror_session {}".format(session_name))["stdout"] + return session_name in output + + +def _get_mirror_gre_type(duthost): + """Return the ERSPAN GRE type expected by the DUT ASIC.""" + identifiers = [ + duthost.facts.get("asic_type", ""), + duthost.facts.get("platform", ""), + duthost.facts.get("hwsku", ""), + ] + normalized_identifiers = [ + _normalize_platform_name(identifier) + for identifier in identifiers + ] + + if any( + "mellanox" in identifier or "nvidia" in identifier + for identifier in normalized_identifiers + ): + return MIRROR_GRE_TYPES["mellanox"] + + for asic_type, gre_type in MIRROR_GRE_TYPES.items(): + normalized_asic_type = _normalize_platform_name(asic_type) + if any( + normalized_asic_type in identifier + for identifier in normalized_identifiers + ): + return gre_type + + return DEFAULT_MIRROR_GRE_TYPE + + +def enable_everflow(duthost): + """Create ERSPAN mirror session and EVERFLOW ACL table.""" + if not _mirror_session_exists(duthost, EVERFLOW_SESSION): + logger.info("Creating Everflow mirror session on %s", duthost.hostname) + duthost.shell( + "config mirror_session add {} {} {} {} {} {} {}".format( + EVERFLOW_SESSION, MIRROR_SRC_IP, MIRROR_DST_IP, MIRROR_DSCP, + MIRROR_TTL, _get_mirror_gre_type(duthost), MIRROR_QUEUE, + ) + ) + + lines = duthost.shell("show acl table EVERFLOW")["stdout_lines"] + if not any("EVERFLOW" in line for line in lines): + ports = _get_acl_bind_ports(duthost) + duthost.shell( + "config acl add table EVERFLOW MIRROR -p {}".format(",".join(ports)) + ) + if not _verify_acl_table_ports(duthost, "EVERFLOW", ports): + raise RuntimeError( + "EVERFLOW not attached to expected interfaces on {}".format( + duthost.hostname + ) + ) + duthost.shell("config save -y") + + +def disable_everflow(duthost): + """Remove EVERFLOW ACL table and mirror session.""" + lines = duthost.shell("show acl table EVERFLOW")["stdout_lines"] + if any("EVERFLOW" in line for line in lines): + duthost.shell("config acl remove table EVERFLOW") + if not _verify_acl_table_removed(duthost, "EVERFLOW"): + raise RuntimeError("EVERFLOW not fully removed on {}".format(duthost.hostname)) + + if _mirror_session_exists(duthost, EVERFLOW_SESSION): + duthost.shell("config mirror_session remove {}".format(EVERFLOW_SESSION)) + duthost.shell("config save -y") + + +# --- EverflowV6 (IPv6 ACL mirror) --- + +EVERFLOWV6_SESSION = "max_tput_ev6_session" +EVERFLOWV6_TABLE = "EVERFLOWV6" + + +def enable_everflowv6(duthost): + """Create ERSPAN mirror session and EVERFLOWV6 ACL table.""" + if not _mirror_session_exists(duthost, EVERFLOWV6_SESSION): + duthost.shell( + "config mirror_session add {} {} {} {} {} {} {}".format( + EVERFLOWV6_SESSION, MIRROR_SRC_IP, MIRROR_DST_IP, + MIRROR_DSCP, + MIRROR_TTL, _get_mirror_gre_type(duthost), MIRROR_QUEUE, + ) + ) + + lines = duthost.shell("show acl table EVERFLOWV6")["stdout_lines"] + if not any("EVERFLOWV6" in line for line in lines): + ports = _get_acl_bind_ports(duthost) + duthost.shell( + "config acl add table EVERFLOWV6 MIRRORV6 -p {}".format(",".join(ports)) + ) + if not _verify_acl_table_ports(duthost, "EVERFLOWV6", ports): + raise RuntimeError( + "EVERFLOWV6 not attached to expected interfaces on {}".format( + duthost.hostname + ) + ) + duthost.shell("config save -y") + + +def disable_everflowv6(duthost): + """Remove EVERFLOWV6 ACL table and mirror session.""" + lines = duthost.shell("show acl table EVERFLOWV6")["stdout_lines"] + if any("EVERFLOWV6" in line for line in lines): + duthost.shell("config acl remove table EVERFLOWV6") + if not _verify_acl_table_removed(duthost, "EVERFLOWV6"): + raise RuntimeError("EVERFLOWV6 not fully removed on {}".format(duthost.hostname)) + + if _mirror_session_exists(duthost, EVERFLOWV6_SESSION): + duthost.shell("config mirror_session remove {}".format(EVERFLOWV6_SESSION)) + duthost.shell("config save -y") + + +# --- IPinIP Decap --- + +IPINIP_DECAP_CONF_TEMPLATE = """[ + {{ + "TUNNEL_DECAP_TERM_TABLE:IPINIP_TUNNEL:{loopback_ip}": {{ + "term_type": "P2MP" + }}, + "OP": "{op}" + }}, + {{ + "TUNNEL_DECAP_TABLE:IPINIP_TUNNEL": {{ + "tunnel_type": "IPINIP", + "dscp_mode": "pipe", + "ecn_mode": "copy_from_outer", + "ttl_mode": "pipe" + }}, + "OP": "{op}" + }} +]""" + + +def _get_loopback_ip(duthost): + """Return the first Loopback0 IPv4 address.""" + result = duthost.shell( + "sonic-db-cli CONFIG_DB keys 'LOOPBACK_INTERFACE|Loopback0|*'" + )["stdout"].strip() + for line in result.splitlines(): + parts = line.split("|") + if len(parts) >= 3: + ip = parts[2].split("/")[0] + if "." in ip: + return ip + raise RuntimeError( + "No IPv4 Loopback0 address found on {}".format(duthost.hostname) + ) + + +def enable_ipinip_decap(duthost): + """Configure IPINIP decap tunnel via swssconfig.""" + loopback_ip = _get_loopback_ip(duthost) + conf = IPINIP_DECAP_CONF_TEMPLATE.format(loopback_ip=loopback_ip, op="SET") + logger.info("Enabling IPinIP decap on %s (loopback=%s)", duthost.hostname, loopback_ip) + + duthost.copy(content=conf, dest="/tmp/ipinip_decap_set.json") + for asic_id in duthost.get_frontend_asic_ids(): + swss = "swss{}".format(asic_id if asic_id is not None else "") + cmds = [ + "docker cp /tmp/ipinip_decap_set.json {}:/ipinip_decap_set.json".format(swss), + "docker exec {} swssconfig /ipinip_decap_set.json".format(swss), + "docker exec {} rm /ipinip_decap_set.json".format(swss), + ] + duthost.shell_cmds(cmds=cmds) + + +def _verify_ipinip_decap_removed(duthost): + """Verify that all IPINIP decap tunnel entries are removed from APP_DB.""" + result = duthost.shell( + 'sonic-db-cli APPL_DB KEYS "TUNNEL_DECAP_TABLE:IPINIP_TUNNEL*"', + module_ignore_errors=True, + )["stdout"].strip() + if result: + logger.warning("IPinIP decap entries still present in APPL_DB: %s", result) + return False + + term_result = duthost.shell( + 'sonic-db-cli APPL_DB KEYS "TUNNEL_DECAP_TERM_TABLE:IPINIP_TUNNEL*"', + module_ignore_errors=True, + )["stdout"].strip() + if term_result: + logger.warning("IPinIP decap term entries still present in APPL_DB: %s", term_result) + return False + + logger.info("All IPinIP decap entries successfully removed from APPL_DB") + return True + + +def disable_ipinip_decap(duthost): + """Remove IPINIP decap tunnel via swssconfig.""" + loopback_ip = _get_loopback_ip(duthost) + conf = IPINIP_DECAP_CONF_TEMPLATE.format(loopback_ip=loopback_ip, op="DEL") + logger.info("Disabling IPinIP decap on %s", duthost.hostname) + + duthost.copy(content=conf, dest="/tmp/ipinip_decap_del.json") + for asic_id in duthost.get_frontend_asic_ids(): + swss = "swss{}".format(asic_id if asic_id is not None else "") + cmds = [ + "docker cp /tmp/ipinip_decap_del.json {}:/ipinip_decap_del.json".format(swss), + "docker exec {} swssconfig /ipinip_decap_del.json".format(swss), + "docker exec {} rm /ipinip_decap_del.json".format(swss), + ] + duthost.shell_cmds(cmds=cmds) + + if not _verify_ipinip_decap_removed(duthost): + raise RuntimeError("IPinIP decap entries were not fully removed on {}".format(duthost.hostname)) + + +def enable_usid_decap(duthost, config): + """Create SRv6 locator and SID for uSID shift-and-forward.""" + srv6_cfg = config.get("srv6", {}) + locator_name = srv6_cfg.get("locator_name", "loc1") + locator_prefix = srv6_cfg.get("locator_prefix", "fcbb:bbbb:1::") + sid_ip = srv6_cfg.get("sid_ip", "fcbb:bbbb:1::") + action = srv6_cfg.get("sid_action", SRv6.uN) + decap_vrf = srv6_cfg.get("decap_vrf", "default") + dscp_mode = srv6_cfg.get("decap_dscp_mode", SRv6.pipe_mode) + + logger.info("Enabling uSID decap on %s (locator=%s)", duthost.hostname, locator_name) + create_srv6_locator( + duthost, + locator_name, + locator_prefix, + block_len=srv6_cfg.get("block_len", 32), + node_len=srv6_cfg.get("node_len", 16), + func_len=srv6_cfg.get("func_len", 0), + arg_len=srv6_cfg.get("arg_len", 0), + ) + create_srv6_sid( + duthost, + locator_name, + sid_ip, + action=action, + decap_vrf=decap_vrf, + decap_dscp_mode=dscp_mode, + ) + duthost.shell("config save -y") + + +def disable_usid_decap(duthost, config): + """Remove SRv6 locator and SID.""" + srv6_cfg = config.get("srv6", {}) + locator_name = srv6_cfg.get("locator_name", "loc1") + sid_ip = srv6_cfg.get("sid_ip", "fcbb:bbbb:1::") + + logger.info("Disabling uSID decap on %s", duthost.hostname) + del_srv6_sid(duthost, locator_name, sid_ip) + del_srv6_locator(duthost, locator_name) + duthost.shell("config save -y") + + +# --------------------------------------------------------------------------- +# DUT config backup/restore +# --------------------------------------------------------------------------- + +CONFIG_DB_PATH = "/etc/sonic/config_db.json" +CONFIG_DB_BACKUP_PATH = "/host/config_db.json.before_max_throughput_test" + + +def backup_dut_config(duthost): + """Save running config and back up config_db.json for later restoration.""" + logger.info("Backing up DUT config on %s", duthost.hostname) + # Persist current running config to config_db.json + duthost.shell("config save -y") + # Copy config_db.json to a backup location + duthost.shell("cp {} {}".format(CONFIG_DB_PATH, CONFIG_DB_BACKUP_PATH)) + logger.info("Config backed up to %s", CONFIG_DB_BACKUP_PATH) + + +def restore_dut_config(duthost): + """Restore config_db.json from backup and reload.""" + logger.info("Restoring DUT config on %s", duthost.hostname) + result = duthost.shell("test -f {}".format(CONFIG_DB_BACKUP_PATH), module_ignore_errors=True) + if result["rc"] != 0: + logger.warning("Backup file %s not found, skipping restore", CONFIG_DB_BACKUP_PATH) + return + duthost.shell("cp {} {}".format(CONFIG_DB_BACKUP_PATH, CONFIG_DB_PATH)) + config_reload( + duthost, + config_source="config_db", + safe_reload=True, + check_intf_up_ports=True, + ) + duthost.shell("rm -f {}".format(CONFIG_DB_BACKUP_PATH)) + + +# --------------------------------------------------------------------------- +# Scenario orchestration +# --------------------------------------------------------------------------- + +# Map feature keys to (enable_fn, disable_fn) pairs. +FEATURE_TOGGLE_MAP = { + "dataacl": (enable_dataacl, disable_dataacl), + "everflow": (enable_everflow, disable_everflow), + "everflowv6": (enable_everflowv6, disable_everflowv6), + "ipinip_decap": (enable_ipinip_decap, disable_ipinip_decap), + "usid_decap": (enable_usid_decap, disable_usid_decap), +} + + +def _call_feature_fn(fn, duthost, feature_key, config): + """Call a feature toggle function, passing config if needed.""" + if feature_key in ("usid_decap",): + fn(duthost, config) + else: + fn(duthost) + + +def cleanup_all_features(duthost, config): + """Disable all features to reach a clean baseline.""" + logger.info("Cleaning all features on %s to reach baseline", duthost.hostname) + for feature_key, (_, disable_fn) in FEATURE_TOGGLE_MAP.items(): + try: + _call_feature_fn(disable_fn, duthost, feature_key, config) + except Exception as e: + logger.warning("Failed to disable %s on %s: %s", feature_key, duthost.hostname, e) + + +def configure_scenario(duthost, scenario_name, config): + """Configure DUT for a scenario: clean baseline then enable required features.""" + scenario = config["scenarios"][scenario_name] + logger.info( + "Configuring scenario '%s' (%s) on %s", + scenario_name, scenario.get("description", ""), duthost.hostname, + ) + + cleanup_all_features(duthost, config) + time.sleep(5) + + enabled_features = [] + for feature_key, (enable_fn, _) in FEATURE_TOGGLE_MAP.items(): + if scenario.get(feature_key, False): + logger.info("Enabling feature: %s", feature_key) + _call_feature_fn(enable_fn, duthost, feature_key, config) + enabled_features.append(feature_key) + + # Allow configs to settle + time.sleep(10) + return enabled_features diff --git a/tests/snappi_tests/dataplane/imports.py b/tests/snappi_tests/dataplane/imports.py index 19e530f2893..48193ba9f4c 100644 --- a/tests/snappi_tests/dataplane/imports.py +++ b/tests/snappi_tests/dataplane/imports.py @@ -183,3 +183,63 @@ # MAC Management # ============================== from snappi_tests.reboot.files.reboot_helper import get_macs # noqa: F403, F401, F405 + +# ============================== +# Exported names +# ============================== +__all__ = [ + # Standard library + "os", "sys", "time", "struct", "json", "yaml", "collections", "logging", + "snappi", "np", + # Third-party + "pytest", "pd", "deepcopy", "tabulate", "natsorted", + "dataclass", "field", "Optional", "Dict", "List", "Any", + "ipaddress", "ip_address", "IPv4Address", "IPv6Address", + "datetime", "IPNetwork", "Final", + # IxNetwork + "TestPlatform", "SessionAssistant", "BatchUpdate", "BatchAdd", "StatViewAssistant", + # Common test utilities + "config_reload", "reboot", "wait_critical_processes", + "wait_until", "wait", "pytest_assert", "pytest_require", "GaugeMetric", + # Topology + "conn_graph_facts", "fanout_graph_facts", "fanout_graph_facts_multidut", + # Snappi framework + "SnappiTestParams", + "snappi_api_serv_ip", "snappi_api_serv_port", "snappi_api", + "snappi_testbed_config", "get_snappi_ports_single_dut", + "get_snappi_ports_multi_dut", "get_snappi_ports", + "cleanup_config", "is_snappi_multidut", "create_ip_list", "__gen_mac", + # QoS + "prio_dscp_map", "all_prio_list", "lossless_prio_list", "lossy_prio_list", + # Snappi helpers + "get_dut_port_id", "wait_for_arp", "fetch_snappi_flow_metrics", + "SnappiFanoutManager", "get_snappi_port_location", "is_traffic_converged", + # Port management + "select_ports", "select_tx_port", "SnappiPortConfig", "SnappiPortType", + # Traffic generation + "generate_background_flows", "generate_pause_flows", "generate_test_flows", + "run_traffic", "setup_base_traffic_config", + "verify_background_flow", "verify_basic_test_flow", + "verify_egress_queue_frame_count", "verify_in_flight_buffer_pkts", + "verify_pause_flow", "verify_pause_frame_count_dut", + "verify_rx_frame_count_dut", "verify_tx_frame_count_dut", + "verify_unset_cev_pause_frame_count", + # Common helpers + "calc_pfc_pause_flow_rate", "config_capture_pkt", "disable_packet_aging", + "get_lossless_buffer_size", "get_pg_dropped_packets", "get_pfc_frame_count", + "packet_capture", "pfc_class_enable_vector", "sec_to_nanosec", + "stop_pfcwd", "traffic_flow_mode", "get_tx_frame_count", "get_rx_frame_count", + "get_egress_queue_count", "config_wred", "enable_ecn", + "config_ingress_lossless_buffer_alpha", "get_peer_snappi_chassis", + "get_addrs_in_subnet", "get_other_hosts_from_ipv6_host", + # PFC/ECN helpers + "run_pfc_test", "skip_warm_reboot", "skip_ecn_tests", + # Packet analysis + "validate_pfc_frame", "is_ecn_marked", "get_ipv4_pkts", + # Variables + "dut_ip_start", "snappi_ip_start", "prefix_length", + "dut_ipv6_start", "snappi_ipv6_start", "v6_prefix_length", + "pfcQueueGroupSize", "pfcQueueValueDict", + # MAC management + "get_macs", +] diff --git a/tests/snappi_tests/dataplane/test_max_throughput_min_pkt_size.py b/tests/snappi_tests/dataplane/test_max_throughput_min_pkt_size.py new file mode 100644 index 00000000000..6dc77729c31 --- /dev/null +++ b/tests/snappi_tests/dataplane/test_max_throughput_min_pkt_size.py @@ -0,0 +1,464 @@ +""" +SRv6 Max Throughput Minimum Packet Size test. + +Sends SRv6 IPv6-in-IPv6 traffic bidirectionally across 32 ports at 100% line rate +and verifies zero drops at known minimum packet sizes per feature-configuration scenario. +""" +import ipaddress + +from tests.snappi_tests.dataplane.imports import * # noqa: F403 +from snappi_tests.dataplane.files.helper import * # noqa: F403 +from snappi_tests.dataplane.files.max_throughput_helper import ( + _load_json_output, + load_max_throughput_config, + get_platform_thresholds, + configure_scenario, + backup_dut_config, + restore_dut_config, +) + +logger = logging.getLogger(__name__) # noqa: F405 + +pytestmark = [pytest.mark.topology("nut")] # noqa: F405 + +MAX_THROUGHPUT_CONFIG = load_max_throughput_config() +SCENARIO_NAMES = list(MAX_THROUGHPUT_CONFIG["scenarios"].keys()) +ROUTE_RANGES = {"IPv6": [[["777:777:777::1", 64, 16]]]} + + +@pytest.mark.parametrize("scenario_name", SCENARIO_NAMES) # noqa: F405 +def test_max_throughput_min_pkt_size( + duthosts, + snappi_api, + get_snappi_ports, + fanout_graph_facts_multidut, + create_snappi_config, + scenario_name, +): + """Verify zero packet loss at 100% line rate for the scenario's minimum packet size.""" + config = MAX_THROUGHPUT_CONFIG + min_ports = config.get("min_ports", 32) + traffic_duration = config.get("traffic_duration_sec", 60) + line_rate = config.get("line_rate_pct", 100) + tolerance_offset = config.get("tolerance_pkt_size_offset", 10) + + pytest_require( # noqa: F405 + len(duthosts) == 1, + "This test requires a single-DUT topology, found {} DUTs".format(len(duthosts)), + ) + duthost = duthosts[0] + + thresholds = get_platform_thresholds(duthost, config) + pytest_require( # noqa: F405 + thresholds is not None, + "Platform '{}' not found in max_throughput_config.yaml — skipping".format( + duthost.facts.get("hwsku", "unknown") + ), + ) + pytest_require( # noqa: F405 + scenario_name in thresholds, + "Scenario '{}' has no threshold for platform '{}'".format( + scenario_name, duthost.facts.get("hwsku", "") + ), + ) + min_pkt_size = thresholds[scenario_name] + + snappi_ports = get_duthost_interface_details( # noqa: F405 + duthosts, get_snappi_ports, "IPv6", protocol_type="bgp" + ) + pytest_require( # noqa: F405 + len(snappi_ports) >= min_ports, + "Need at least {} snappi ports, only {} available — skipping".format( + min_ports, len(snappi_ports) + ), + ) + snappi_ports = snappi_ports[:min_ports] + half = len(snappi_ports) // 2 + tx_ports = snappi_ports[:half] + rx_ports = snappi_ports[half:] + + backup_dut_config(duthost) + + try: + configure_scenario(duthost, scenario_name, config) + + logger.info( + "Scenario '%s': testing min packet size %dB at %d%% line rate for %ds", + scenario_name, min_pkt_size, line_rate, traffic_duration, + ) + + _build_and_run_traffic( + snappi_api, + create_snappi_config, + snappi_ports, + tx_ports, + rx_ports, + frame_size=min_pkt_size, + line_rate=line_rate, + duration_sec=traffic_duration, + scenario_name=scenario_name, + max_loss_pct=0.0, + config=config, + duthost=duthost, + ) + + tolerance_pkt_size = min_pkt_size + tolerance_offset + logger.info( + "Scenario '%s': tolerance check at %dB (min + %dB offset)", + scenario_name, tolerance_pkt_size, tolerance_offset, + ) + + _build_and_run_traffic( + snappi_api, + create_snappi_config, + snappi_ports, + tx_ports, + rx_ports, + frame_size=tolerance_pkt_size, + line_rate=line_rate, + duration_sec=traffic_duration, + scenario_name=scenario_name, + max_loss_pct=0.001, + config=config, + duthost=duthost, + ) + + finally: + logger.info("Restoring DUT config after scenario '%s'", scenario_name) + restore_dut_config(duthost) + + +def _build_and_run_traffic( + snappi_api, + create_snappi_config, + snappi_ports, + tx_ports, + rx_ports, + frame_size, + line_rate, + duration_sec, + scenario_name, + max_loss_pct, + config, + duthost, +): + """Build fresh snappi config with SRv6 IPv6-in-IPv6 flows, run traffic, and assert loss.""" + flow_name = "max_tput_{}_{}B".format(scenario_name, frame_size) + srv6_cfg = config.get("srv6", {}) + validate_srv6_stats = config["scenarios"][scenario_name].get( + "usid_decap", False + ) + + snappi_extra_params = SnappiTestParams() # noqa: F405 + ranges = _generate_unique_route_ranges("IPv6", len(snappi_ports)) + snappi_extra_params.protocol_config = { + "Tx": { + "route_ranges": ranges, + "protocol_type": "bgp", + "ports": tx_ports, + "subnet_type": "IPv6", + "is_rdma": False, + }, + "Rx": { + "route_ranges": ranges, + "protocol_type": "bgp", + "ports": rx_ports, + "subnet_type": "IPv6", + "is_rdma": False, + }, + } + + snappi_config, snappi_obj_handles = create_snappi_config(snappi_extra_params) + + snappi_extra_params.traffic_flow_config = [ + { + "line_rate": line_rate, + "frame_size": frame_size, + "is_rdma": False, + "flow_name": flow_name, + "tx_names": snappi_obj_handles["Tx"]["network_group"], + "rx_names": snappi_obj_handles["Rx"]["network_group"], + "mesh_type": "mesh", + } + ] + + snappi_config = create_traffic_items(snappi_config, snappi_extra_params) # noqa: F405 + snappi_api.set_config(snappi_config) + + start_stop(snappi_api, operation="start", op_type="protocols") # noqa: F405 + check_bgp_state(snappi_api, "IPv6") # noqa: F405 + + ixnet = snappi_api._ixnetwork + ixnet.Traffic.TrafficItem.find().update(BiDirectional=True, SrcDestMesh="fullMesh") + + _apply_srv6_packet_headers(ixnet, srv6_cfg) + _clear_dut_counters(duthost) + srv6_counters_before = ( + _get_srv6_mysid_counters(duthost, srv6_cfg) + if validate_srv6_stats else None + ) + + start_stop(snappi_api, operation="start", op_type="traffic") # noqa: F405 + + time.sleep(5) # noqa: F405 + if validate_srv6_stats: + _verify_srv6_mysid_counter_delta( + duthost, srv6_cfg, srv6_counters_before + ) + + logger.info("Traffic running for %d seconds ...", duration_sec) + wait_with_message("Running traffic for", duration_sec) # noqa: F405 + + start_stop(snappi_api, operation="stop", op_type="traffic") # noqa: F405 + _verify_no_dut_drops(duthost, snappi_ports) + + df = get_stats( # noqa: F405 + snappi_api, "Traffic Item Statistics", columns=None, return_type="df" + ) + df = df[["name", "frames_tx", "frames_rx", "loss"]] + df[["loss"]] = pd.to_numeric(df["loss"], errors="coerce") # noqa: F405 + df["Status"] = (df["loss"] <= max_loss_pct).map({True: "PASS", False: "FAIL"}) + + logger.info( + "Scenario '%s' @ %dB results:\n%s", + scenario_name, + frame_size, + tabulate(df, headers="keys", tablefmt="psql", showindex=False), # noqa: F405 + ) + + max_loss = df["loss"].max() + + start_stop(snappi_api, operation="stop", op_type="protocols") # noqa: F405 + + pytest_assert( # noqa: F405 + max_loss <= max_loss_pct, + "Scenario '{}' FAILED at {}B: loss {:.4f}% exceeds threshold {:.4f}%".format( + scenario_name, frame_size, max_loss, max_loss_pct, + ), + ) + logger.info("Scenario '%s' PASSED at %dB (loss %.4f%%)", scenario_name, frame_size, max_loss) + + +def _generate_unique_route_ranges(ip_version, port_count): + """Generate one unique advertised route prefix per Snappi port.""" + base_ip, prefix, route_count = ROUTE_RANGES[ip_version][0][0] + ip_class = ( + ipaddress.IPv4Address + if ip_version == "IPv4" else ipaddress.IPv6Address + ) + base_ip = int(ip_class(base_ip)) + increment = route_count if ip_version == "IPv4" else 1 << 96 + return [ + [[str(ip_class(base_ip + count * increment)), prefix, route_count]] + for count in range(port_count) + ] + + +def _counter_to_int(value): + """Convert SONiC CLI counter output to an integer.""" + if value in (None, "", "N/A"): + return 0 + return int(str(value).replace(",", "")) + + +def _get_srv6_mysid_counters(duthost, srv6_cfg): + """Read packet and byte counters for the configured SRv6 mySID.""" + sid_ip = srv6_cfg.get("sid_ip", "fcbb:bbbb:1::") + sid_with_prefix = "{}/{}".format( + sid_ip, srv6_cfg.get("sid_prefix_len", 48) + ) + stats_list = duthost.show_and_parse("show srv6 stats") + + for stats in stats_list: + mysid = stats.get("mysid", "") + if ( + mysid in (sid_ip, sid_with_prefix) or + mysid.startswith("{}/".format(sid_ip)) + ): + return { + "packets": _counter_to_int(stats.get("packets")), + "bytes": _counter_to_int(stats.get("bytes")), + } + + pytest_assert( # noqa: F405 + False, + "SRv6 mySID {} not found in 'show srv6 stats': {}".format( + sid_ip, stats_list + ), + ) + + +def _verify_srv6_mysid_counter_delta(duthost, srv6_cfg, before): + """Verify SRv6 mySID counters increment while traffic is running.""" + after = _get_srv6_mysid_counters(duthost, srv6_cfg) + pytest_assert( # noqa: F405 + after["packets"] > before["packets"] and + after["bytes"] > before["bytes"], + "SRv6 mySID counters did not increment. Before: {}, after: {}".format( + before, after + ), + ) + logger.info( + "SRv6 mySID counters incremented. Before: %s, after: %s", + before, + after, + ) + + +def _get_dut_ports(snappi_ports): + """Return unique DUT front-panel ports used by the Snappi test.""" + return sorted({port["peer_port"] for port in snappi_ports}) + + +def _clear_dut_counters(duthost): + """Clear DUT port and queue counters before traffic starts.""" + duthost.shell("sonic-clear counters") + duthost.shell("sonic-clear queuecounters") + + +def _verify_no_port_drops(duthost, dut_ports): + """Verify no RX/TX drops on DUT ports used by the test.""" + output = duthost.shell( + "portstat -i {} -j".format(",".join(dut_ports)) + )["stdout"] + stats = _load_json_output(output) + drops = [] + + for port in dut_ports: + port_stats = stats.get(port, {}) + pytest_assert( # noqa: F405 + port_stats, + "No portstat counters found for {}".format(port), + ) + for counter in ("RX_DRP", "TX_DRP"): + value = _counter_to_int(port_stats.get(counter)) + if value: + drops.append("{} {}={}".format(port, counter, value)) + + pytest_assert( # noqa: F405 + not drops, + "Unexpected DUT port drops: {}".format(", ".join(drops)), + ) + + +def _verify_no_queue_drops(duthost, dut_ports): + """Verify no queue drops on DUT ports used by the test.""" + drops = [] + output = duthost.shell("show queue counters --all -j")["stdout"] + queue_stats = _load_json_output(output) + + for port in dut_ports: + port_queues = queue_stats.get(port, {}) + pytest_assert( # noqa: F405 + port_queues, + "No queue counters found for {}".format(port), + ) + + for queue, queue_stats in port_queues.items(): + if not isinstance(queue_stats, dict): + continue + drop_packets = _counter_to_int(queue_stats.get("droppacket")) + if drop_packets: + drops.append( + "{} {} droppacket={}".format(port, queue, drop_packets) + ) + + pytest_assert( # noqa: F405 + not drops, + "Unexpected DUT queue drops: {}".format(", ".join(drops)), + ) + + +def _verify_no_dut_drops(duthost, snappi_ports): + """Verify DUT port and queue counters did not record drops.""" + dut_ports = _get_dut_ports(snappi_ports) + _verify_no_port_drops(duthost, dut_ports) + _verify_no_queue_drops(duthost, dut_ports) + + +def _apply_srv6_packet_headers(ixnet, srv6_cfg): + """Add SRv6 IPv6-in-IPv6 encapsulation to the IxNetwork traffic items.""" + outer_dst_ip = srv6_cfg.get("outer_dst_ip", "fcbb:bbbb:1:11a::2") + + traffic_items = ixnet.Traffic.TrafficItem.find() + configured_elements = 0 + for ti in traffic_items: + config_elements = ti.ConfigElement.find() + for ce in config_elements: + stacks = ce.Stack.find() + ipv6_stacks = [ + s for s in stacks + if "IPv6" in s.DisplayName or "ipv6" in s.StackTypeId + ] + pytest_assert( # noqa: F405 + ipv6_stacks, "Traffic item has no outer IPv6 stack" + ) + + outer_ipv6 = ipv6_stacks[0] + outer_dst_set = False + outer_next_header_set = False + for field in outer_ipv6.Field.find(): + if "Destination Address" in field.DisplayName: + field.SingleValue = outer_dst_ip + outer_dst_set = True + elif "Next Header" in field.DisplayName: + field.SingleValue = "41" # IPv6-in-IPv6 + outer_next_header_set = True + pytest_assert( # noqa: F405 + outer_dst_set, "Outer IPv6 destination field was not found" + ) + pytest_assert( # noqa: F405 + outer_next_header_set, + "Outer IPv6 next-header field was not found", + ) + + proto_template = ixnet.Traffic.ProtocolTemplate.find( + StackTypeId="ipv6" + ) + pytest_assert( # noqa: F405 + proto_template, + "IxNetwork IPv6 protocol template was not found", + ) + if len(ipv6_stacks) < 2: + outer_ipv6.Append(proto_template) + stacks = ce.Stack.find() + ipv6_stacks = [ + s for s in stacks + if "IPv6" in s.DisplayName or "ipv6" in s.StackTypeId + ] + + pytest_assert( + len(ipv6_stacks) >= 2, + "Traffic item has no inner IPv6 stack after SRv6 header " + "configuration", + ) # noqa: F405 + inner_ipv6 = ipv6_stacks[1] + inner_src_set = False + inner_dst_set = False + for field in inner_ipv6.Field.find(): + if "Source Address" in field.DisplayName: + field.ValueType = "increment" + field.StartValue = "2001:db8:1::1" + field.StepValue = "::1" + field.CountValue = "1000" + inner_src_set = True + elif "Destination Address" in field.DisplayName: + field.ValueType = "increment" + field.StartValue = "2001:db8:2::1" + field.StepValue = "::1" + field.CountValue = "1000" + inner_dst_set = True + + pytest_assert( # noqa: F405 + inner_src_set, "Inner IPv6 source field was not found" + ) + pytest_assert( # noqa: F405 + inner_dst_set, "Inner IPv6 destination field was not found" + ) + configured_elements += 1 + + pytest_assert( # noqa: F405 + configured_elements > 0, + "No SRv6 traffic config elements were updated", + ) + traffic_items.Generate() diff --git a/tests/snappi_tests/ecn/test_dequeue_ecn_with_snappi.py b/tests/snappi_tests/ecn/test_dequeue_ecn_with_snappi.py index 225d61f771a..4e090e79dd2 100644 --- a/tests/snappi_tests/ecn/test_dequeue_ecn_with_snappi.py +++ b/tests/snappi_tests/ecn/test_dequeue_ecn_with_snappi.py @@ -26,7 +26,7 @@ # sent to see the marked packets at the end of the flow. ECN_PARAMS_BY_ASIC = { 'default': {'ecn_params': {'kmin': 50000, 'kmax': 51000, 'pmax': 100}, 'pkt_count': 101}, - 'broadcom': {'ecn_params': {'kmin': 40000, 'kmax': 51000, 'pmax': 100}, 'pkt_count': 301}, + 'broadcom': {'ecn_params': {'kmin': 40000, 'kmax': 168000, 'pmax': 100}, 'pkt_count': 301}, } diff --git a/tests/snappi_tests/pfc/files/m2o_fluctuating_lossless_helper.py b/tests/snappi_tests/pfc/files/m2o_fluctuating_lossless_helper.py index 659f964c8a7..7dbc854082a 100644 --- a/tests/snappi_tests/pfc/files/m2o_fluctuating_lossless_helper.py +++ b/tests/snappi_tests/pfc/files/m2o_fluctuating_lossless_helper.py @@ -150,6 +150,31 @@ def _add_demand(prio, rate): return sum(losses) / len(losses) +def get_expected_total_drop_percent(test_flow_rate_percent, + bg_prio_list, + bg_flow_rate_percent, + expected_bg_loss_percent): + """Compute expected overall drop percent from offered-load mix. + + The total drop is background-offered-share multiplied by the expected + background loss percent, normalized by total offered load. + """ + pytest_assert(bg_flow_rate_percent, "FAIL: bg_flow_rate_percent must be non-empty") + pytest_assert( + len(set(bg_flow_rate_percent)) == 1, + "FAIL: bg_flow_rate_percent entries must be equal, got {}".format(bg_flow_rate_percent)) + + bg_flow_count = len(bg_prio_list) + bg_rate_per_flow = bg_flow_rate_percent[0] + total_bg_offered_percent = bg_flow_count * bg_rate_per_flow + total_test_offered_percent = sum(test_flow_rate_percent) + total_offered_percent = total_bg_offered_percent + total_test_offered_percent + pytest_assert(total_offered_percent > 0, + "FAIL: total offered percent must be > 0 (got {})".format(total_offered_percent)) + + return total_bg_offered_percent * expected_bg_loss_percent / total_offered_percent + + def run_m2o_fluctuating_lossless_test(api, testbed_config, port_config_list, @@ -277,8 +302,6 @@ def run_m2o_fluctuating_lossless_test(api, pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[egress_duthost.hostname][dut_tx_port]['tx_drp'] drop_percentage = (100 * pkt_drop) / total_rx_pkts - pytest_assert(abs(drop_percentage - 8) < 1, 'FAIL: Drop packets must be around 8 percent') - expected_bg_loss_percent = get_expected_bg_loss_percent( egress_duthost=egress_duthost, test_prio_list=test_prio_list, @@ -287,6 +310,18 @@ def run_m2o_fluctuating_lossless_test(api, bg_flow_rate_percent=BG_FLOW_AGGR_RATE_PERCENT, asic_value=rx_port.get('asic_value'), port=dut_tx_port) + + expected_drop_percentage = get_expected_total_drop_percent( + test_flow_rate_percent=TEST_FLOW_AGGR_RATE_PERCENT, + bg_prio_list=bg_prio_list, + bg_flow_rate_percent=BG_FLOW_AGGR_RATE_PERCENT, + expected_bg_loss_percent=expected_bg_loss_percent) + + pytest_assert( + abs(drop_percentage - expected_drop_percentage) < 1, + 'FAIL: Drop packets must be around {:.2f}% (got {:.2f}%)'.format( + expected_drop_percentage, drop_percentage)) + logger.info('Expected per-Background-Flow loss: {:.2f}% (tolerance +/- {}%)'.format( expected_bg_loss_percent, BG_LOSS_TOLERANCE_PERCENT)) diff --git a/tests/snappi_tests/unit_tests/pfc/unit_test_m2o_fluctuating_lossless_helper.py b/tests/snappi_tests/unit_tests/pfc/unit_test_m2o_fluctuating_lossless_helper.py index 0b969ab8a39..f9771cafcbf 100644 --- a/tests/snappi_tests/unit_tests/pfc/unit_test_m2o_fluctuating_lossless_helper.py +++ b/tests/snappi_tests/unit_tests/pfc/unit_test_m2o_fluctuating_lossless_helper.py @@ -37,7 +37,10 @@ MODULE_PATH = (Path(__file__).resolve().parents[3] / "snappi_tests/pfc/files/m2o_fluctuating_lossless_helper.py") -FUNCTION_NAMES = ("get_expected_bg_loss_percent",) +FUNCTION_NAMES = ( + "get_expected_bg_loss_percent", + "get_expected_total_drop_percent", +) def _load_functions(names): @@ -174,3 +177,28 @@ def test_expected_bg_loss_matches_analytical_dwrr_split( ) assert result == pytest.approx(expected_loss, abs=0.01) + + +@pytest.mark.parametrize( + "test_rate,bg_prio,bg_rates,bg_loss,expected_total_drop", + [ + # m2o fluctuating-lossless default: total offered = 110%, + # bg offered = 80%, bg loss ~= 11.2676% => total drop ~= 8.1946% + ([20, 10], [0, 1, 2, 5], [20, 20, 20, 20], 11.2676, 8.1946), + # Uniform-weight reference: bg loss = 10% => total drop ~= 7.2727% + ([20, 10], [0, 1, 2, 5], [20, 20, 20, 20], 10.0, 7.2727), + # Mix-weight from Cisco-8102: bg_prio_list=[2,1,5,6] + ([20, 10], [2, 1, 5, 6], [20, 20, 20, 20], 11.2676, 8.1946), + ], +) +def test_expected_total_drop_percent_matches_weighted_formula( + helper_ns, test_rate, bg_prio, bg_rates, bg_loss, expected_total_drop): + """Overall drop must equal weighted BG-loss contribution over total load.""" + result = helper_ns["get_expected_total_drop_percent"]( + test_flow_rate_percent=test_rate, + bg_prio_list=bg_prio, + bg_flow_rate_percent=bg_rates, + expected_bg_loss_percent=bg_loss, + ) + + assert result == pytest.approx(expected_total_drop, abs=0.01) diff --git a/tests/srv6/test_srv6_vlan_forwarding.py b/tests/srv6/test_srv6_vlan_forwarding.py index 142d4370479..9a327cc3767 100644 --- a/tests/srv6/test_srv6_vlan_forwarding.py +++ b/tests/srv6/test_srv6_vlan_forwarding.py @@ -24,6 +24,7 @@ def run_srv6_downstrean_traffic_test(duthost, dut_mac, ptf_src_port, ptf_dst_port, neighbor_ip, ptfadapter, ptfhost, with_srh): + ptf_mgmt_ipv6 = ptfhost.mgmt_ipv6 if ptfhost.mgmt_ipv6 else "1000::1" for i in range(0, 10): # generate a random payload payload = ''.join(random.choices(string.ascii_letters + string.digits, k=20)) @@ -31,7 +32,7 @@ def run_srv6_downstrean_traffic_test(duthost, dut_mac, ptf_src_port, ptf_dst_por injected_pkt = simple_ipv6_sr_packet( eth_dst=dut_mac, eth_src=ptfadapter.dataplane.get_mac(0, ptf_src_port).decode(), - ipv6_src=ptfhost.mgmt_ipv6, + ipv6_src=ptf_mgmt_ipv6, ipv6_dst="fcbb:bbbb:1:2::", srh_seg_left=1, srh_nh=41, @@ -39,7 +40,7 @@ def run_srv6_downstrean_traffic_test(duthost, dut_mac, ptf_src_port, ptf_dst_por ) else: injected_pkt = Ether(dst=dut_mac, src=ptfadapter.dataplane.get_mac(0, ptf_src_port).decode()) \ - / IPv6(src=ptfhost.mgmt_ipv6, dst="fcbb:bbbb:1:2::") \ + / IPv6(src=ptf_mgmt_ipv6, dst="fcbb:bbbb:1:2::") \ / IPv6() / UDP(dport=4791) / Raw(load=payload) expected_pkt = injected_pkt.copy() @@ -222,11 +223,12 @@ def test_srv6_uN_no_vlan_flooding(setup_downstream_uN, proxy_arp_enabled, ptfada ptfadapter.dataplane.flush() # generate a random payload payload = ''.join(random.choices(string.ascii_letters + string.digits, k=20)) + ptf_mgmt_ipv6 = ptfhost.mgmt_ipv6 if ptfhost.mgmt_ipv6 else "1000::1" if with_srh: injected_pkt = simple_ipv6_sr_packet( eth_dst=dut_mac, eth_src=ptfadapter.dataplane.get_mac(0, ptf_src_port).decode(), - ipv6_src=ptfhost.mgmt_ipv6, + ipv6_src=ptf_mgmt_ipv6, ipv6_dst="fcbb:bbbb:1:2::", srh_seg_left=0, srh_nh=41, @@ -234,7 +236,7 @@ def test_srv6_uN_no_vlan_flooding(setup_downstream_uN, proxy_arp_enabled, ptfada ) else: injected_pkt = Ether(dst=dut_mac, src=ptfadapter.dataplane.get_mac(0, ptf_src_port).decode()) \ - / IPv6(src=ptfhost.mgmt_ipv6, dst="fcbb:bbbb:1:2::") \ + / IPv6(src=ptf_mgmt_ipv6, dst="fcbb:bbbb:1:2::") \ / IPv6() / UDP(dport=4791) / Raw(load=payload) expected_pkt = injected_pkt.copy() diff --git a/tests/test_pretest.py b/tests/test_pretest.py index 0ba9800a9ed..fbc2ca1588a 100644 --- a/tests/test_pretest.py +++ b/tests/test_pretest.py @@ -524,13 +524,30 @@ def test_update_saithrift_ptf(request, ptfhost, duthosts, enum_dut_hostname): if result["failed"] or "OK" not in result["msg"]: pytest.fail("Download failed/error while installing python saithrift package: {}".format(py_saithrift_url)) ptfhost.shell("dpkg -i {}".format(os.path.join("/root", pkg_name))) - # In 202405 branch, the switch_sai_thrift package is inside saithrift-0.9-py3.11.egg - # We need to move it out to the correct location - PY_PATH = "/usr/lib/python3/dist-packages/" - SRC_PATH = PY_PATH + "saithrift-0.9-py3.11.egg/switch_sai_thrift" - DST_PATH = PY_PATH + "switch_sai_thrift" - if ptfhost.stat(path=SRC_PATH)['stat']['exists'] and not ptfhost.stat(path=DST_PATH)['stat']['exists']: - ptfhost.copy(src=SRC_PATH, dest=PY_PATH, remote_src=True) + # The saithrift deb installs switch_sai_thrift inside an egg dir named for the + # Python version it was built against (e.g. saithrift-0.9-py3.11.egg on + # bookworm/OS12, saithrift-0.9-py3.13.egg on trixie/OS13). The PTF runs from its + # own virtualenv (/root/env-python3) whose sys.path references that egg dir via + # easy-install.pth; dpkg does NOT update the .pth when the egg version changes, + # so switch_sai_thrift becomes unimportable when the image's Python version + # differs from the PTF's (e.g. a trixie/py3.13 deb on a bookworm/py3.11 PTF). + # Locate the actual egg (any py3.x) and copy switch_sai_thrift into the PTF + # virtualenv's site-packages, which is always on the runner's sys.path. + PY_PATH = "/usr/lib/python3/dist-packages" + egg_switch_sai = ptfhost.shell( + "ls -d {}/saithrift-0.9-py3.*.egg/switch_sai_thrift 2>/dev/null | head -1".format(PY_PATH), + module_ignore_errors=True, + )["stdout"].strip() + if egg_switch_sai: + # Prefer the PTF virtualenv site-packages (always on the ptf_runner sys.path); + # fall back to the system dist-packages if the virtualenv is absent. + site_dir = ptfhost.shell( + "/root/env-python3/bin/python -c " + "'import sysconfig; print(sysconfig.get_paths()[\"purelib\"])'", + module_ignore_errors=True, + )["stdout"].strip() or PY_PATH + ptfhost.shell("rm -rf {}/switch_sai_thrift".format(site_dir), module_ignore_errors=True) + ptfhost.copy(src=egg_switch_sai, dest=site_dir + "/", remote_src=True) logging.info("Python saithrift package installed successfully") diff --git a/tests/transceiver/attribute_parser/attribute_keys.py b/tests/transceiver/attribute_parser/attribute_keys.py index feb8eb67972..47225569ff1 100644 --- a/tests/transceiver/attribute_parser/attribute_keys.py +++ b/tests/transceiver/attribute_parser/attribute_keys.py @@ -14,7 +14,7 @@ SYSTEM_ATTRIBUTES_KEY = "SYSTEM_ATTRIBUTES" PHYSICAL_OIR_ATTRIBUTES_KEY = "PHYSICAL_OIR_ATTRIBUTES" REMOTE_RESEAT_ATTRIBUTES_KEY = "REMOTE_RESEAT_ATTRIBUTES" -CDB_FW_UPGRADE_ATTRIBUTES_KEY = "CDB_FW_UPGRADE_ATTRIBUTES" +CDB_FIRMWARE_UPGRADE_ATTRIBUTES_KEY = "CDB_FIRMWARE_UPGRADE_ATTRIBUTES" DOM_ATTRIBUTES_KEY = "DOM_ATTRIBUTES" VDM_ATTRIBUTES_KEY = "VDM_ATTRIBUTES" PM_ATTRIBUTES_KEY = "PM_ATTRIBUTES" diff --git a/tests/transceiver/attribute_parser/attribute_manager.py b/tests/transceiver/attribute_parser/attribute_manager.py index 7d42c5f9213..1b317e1666f 100644 --- a/tests/transceiver/attribute_parser/attribute_manager.py +++ b/tests/transceiver/attribute_parser/attribute_manager.py @@ -324,7 +324,7 @@ def _resolve_priority(tree, base_attrs, dut_name, platform, hwsku): # ``active_firmware_version`` is a reserved BASE_ATTRIBUTES key: # ``DutInfoLoader`` does not currently populate it, so # ``firmware_overrides`` shards are silently inert until the producer - # (planned to live with the CDB FW upgrade test infrastructure) + # (planned to live with the CDB firmware upgrade test infrastructure) # populates this key per port. The slot is reserved in the schema so # contributors do not invent ad-hoc keys for firmware-conditional # overrides; the resolver fail-safes to the empty layer when the key diff --git a/tests/transceiver/common/prerequisites.py b/tests/transceiver/common/prerequisites.py index 8e80b3550e9..d1b4bac50a2 100644 --- a/tests/transceiver/common/prerequisites.py +++ b/tests/transceiver/common/prerequisites.py @@ -7,7 +7,7 @@ from tests.common.platform.interface_utils import get_dut_interfaces_status from tests.transceiver.attribute_parser.attribute_keys import ( - CDB_FW_UPGRADE_ATTRIBUTES_KEY, + CDB_FIRMWARE_UPGRADE_ATTRIBUTES_KEY, EEPROM_ATTRIBUTES_KEY, ) from tests.transceiver.utils.cli_parser_helper import parse_eeprom @@ -154,7 +154,7 @@ def check_gold_firmware(duthost, port_attributes_dict): A port is in scope iff ``EEPROM_ATTRIBUTES.cmis_active_optical`` is True. For every in-scope port: - * ``CDB_FW_UPGRADE_ATTRIBUTES.gold_firmware_version`` MUST be defined - + * ``CDB_FIRMWARE_UPGRADE_ATTRIBUTES.gold_firmware_version`` MUST be defined - a missing value is a failure (inventory gap). * the active firmware reported by the CLI MUST equal that value - otherwise the port is a failure (FW mismatch). @@ -207,8 +207,8 @@ def check_gold_firmware(duthost, port_attributes_dict): matched = [] failures = [] for port in in_scope: - cdb_fw_attrs = port_attributes_dict[port].get(CDB_FW_UPGRADE_ATTRIBUTES_KEY, {}) - expected_fw = cdb_fw_attrs.get("gold_firmware_version") + cdb_firmware_attrs = port_attributes_dict[port].get(CDB_FIRMWARE_UPGRADE_ATTRIBUTES_KEY, {}) + expected_fw = cdb_firmware_attrs.get("gold_firmware_version") if not expected_fw: failures.append(f"{port}: gold_firmware_version not configured") logger.warning("Port %s: cmis_active_optical=True but gold_firmware_version missing", port) diff --git a/tests/transceiver/conftest.py b/tests/transceiver/conftest.py index 90e5d35f640..193aee3654f 100644 --- a/tests/transceiver/conftest.py +++ b/tests/transceiver/conftest.py @@ -290,7 +290,7 @@ def gold_fw_verified(duthost, port_attributes_dict): """Gate: every CMIS active-optical transceiver runs its gold firmware. A port is in scope iff its ``EEPROM_ATTRIBUTES.cmis_active_optical`` is - True; for those ports ``CDB_FW_UPGRADE_ATTRIBUTES.gold_firmware_version`` + True; for those ports ``CDB_FIRMWARE_UPGRADE_ATTRIBUTES.gold_firmware_version`` MUST be configured AND must match the active firmware reported by the CLI. Other ports are out of scope (no expectation to compare against). diff --git a/tests/transceiver/eeprom/test_eeprom_basic.py b/tests/transceiver/eeprom/test_eeprom_basic.py index a27126afcbc..7169b8872a6 100644 --- a/tests/transceiver/eeprom/test_eeprom_basic.py +++ b/tests/transceiver/eeprom/test_eeprom_basic.py @@ -3,7 +3,7 @@ from tests.transceiver.utils.cli_parser_helper import parse_eeprom from tests.transceiver.attribute_parser.attribute_keys import ( BASE_ATTRIBUTES_KEY, - CDB_FW_UPGRADE_ATTRIBUTES_KEY, + CDB_FIRMWARE_UPGRADE_ATTRIBUTES_KEY, EEPROM_ATTRIBUTES_KEY, ) @@ -59,7 +59,7 @@ def test_eeprom_content_verification_via_show_cli(duthost, port_attributes_dict) base_attrs = port_attrs.get(BASE_ATTRIBUTES_KEY, {}) eeprom_attrs = port_attrs.get(EEPROM_ATTRIBUTES_KEY, {}) - cmis_fw_attrs = port_attrs.get(CDB_FW_UPGRADE_ATTRIBUTES_KEY, {}) + cdb_firmware_attrs = port_attrs.get(CDB_FIRMWARE_UPGRADE_ATTRIBUTES_KEY, {}) field_failures = [] for cli_key, attr_key in EEPROM_EXPECTED_CLI_KEY_TO_TRANSCEIVER_INV_KEY_MAPPING.items(): @@ -68,7 +68,7 @@ def test_eeprom_content_verification_via_show_cli(duthost, port_attributes_dict) elif attr_key in eeprom_attrs: expected_value = eeprom_attrs.get(attr_key) else: - expected_value = cmis_fw_attrs.get(attr_key) + expected_value = cdb_firmware_attrs.get(attr_key) if expected_value is None: continue diff --git a/tests/upgrade_path/test_upgrade_gnoi.py b/tests/upgrade_path/test_upgrade_gnoi.py index 981155d3785..a29c6143f8b 100644 --- a/tests/upgrade_path/test_upgrade_gnoi.py +++ b/tests/upgrade_path/test_upgrade_gnoi.py @@ -1,7 +1,7 @@ import logging import pytest -from tests.common.fixtures.grpc_fixtures import gnmi_tls # noqa: F401 +from tests.common.fixtures.grpc_fixtures import gnmi_tls, ptf_grpc, ptf_gnoi, setup_gnoi_tls_server # noqa: F401 from tests.upgrade_path.test_upgrade_path import setup_upgrade_test from tests.common.helpers.upgrade_helpers import perform_gnoi_upgrade, GnoiUpgradeConfig @@ -29,7 +29,7 @@ def gnoi_upgrade_path_lists(request): def test_upgrade_via_gnoi( localhost, duthosts, ptfhost, rand_one_dut_hostname, nbrhosts, fanouthosts, tbinfo, request, - gnoi_upgrade_path_lists, ptf_gnoi, # noqa: F811 + gnoi_upgrade_path_lists, gnmi_tls, ptf_gnoi, # noqa: F811 conn_graph_facts, xcvr_skip_list ): duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/voq/test_voq_chassis_app_db_consistency.py b/tests/voq/test_voq_chassis_app_db_consistency.py index d330f9f560f..7de72c731af 100644 --- a/tests/voq/test_voq_chassis_app_db_consistency.py +++ b/tests/voq/test_voq_chassis_app_db_consistency.py @@ -247,7 +247,8 @@ def verify_pc_update(): # Recover all states if test_case == "config_reload_with_config_save": logger.info("Restore config from minigraph.") - config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True) + config_reload(duthost, config_source='minigraph', safe_reload=True, + check_intf_up_ports=True, override_config=True) wait_critical_processes(duthost) pytest_assert(wait_until(300, 20, 0, check_interface_status_of_up_ports, duthost), "Not all ports that are admin up on are operationally up") diff --git a/tests/voq/test_voq_intfs.py b/tests/voq/test_voq_intfs.py index a5c99001cc8..d4fd5209e42 100644 --- a/tests/voq/test_voq_intfs.py +++ b/tests/voq/test_voq_intfs.py @@ -113,7 +113,8 @@ def test_cycle_voq_intf(duthosts, all_cfg_facts, nbrhosts, nbr_macs): finally: # restore interface from minigraph logger.info("Restore config from minigraph.") - config_reload(duthost, config_source='minigraph', safe_reload=True, check_intf_up_ports=True) + config_reload(duthost, config_source='minigraph', safe_reload=True, + check_intf_up_ports=True, override_config=True) pytest_assert(wait_until(300, 10, 0, check_bgp_neighbors, duthosts), "All BGP's are not established after config reload from original minigraph") duthost.shell_cmds(cmds=["config save -y"]) diff --git a/tests/vrf/test_vrf.py b/tests/vrf/test_vrf.py index dfbbd244188..a4a4037ab8c 100644 --- a/tests/vrf/test_vrf.py +++ b/tests/vrf/test_vrf.py @@ -1730,6 +1730,18 @@ def restore_vrf(self, duthost): for ip in ips: duthost.shell("config interface ip add {} {}".format(intf, ip)) + time.sleep(5) + + for intf, ip_facts in list(g_vars["vrf_intfs"]["Vrf1"].items()): + if 'Vlan' not in intf: + continue + for ver, ips in list(ip_facts.items()): + for ip in ips: + neigh_ip = ip.ip + 1 + ping_cmd = 'ping' if ip.version == 4 else 'ping6' + duthost.shell("{} -I Vrf1 {} -c 1 -f -W1".format(ping_cmd, + neigh_ip), module_ignore_errors=True) + @pytest.fixture(scope="class", autouse=True) def setup_vrf_deletion(self, duthosts, rand_one_dut_hostname, ptfhost, tbinfo, cfg_facts): duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/vxlan/conftest.py b/tests/vxlan/conftest.py index e20c3577bb8..973553217a1 100644 --- a/tests/vxlan/conftest.py +++ b/tests/vxlan/conftest.py @@ -354,3 +354,16 @@ def restore_config_by_config_reload(duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] logger.info("Restore config after running tests") config_reload(duthost, safe_reload=True) + + +@pytest.fixture(autouse=True) +def ignore_expected_loganalyzer_exception(duthost, loganalyzer): + if loganalyzer: + # The following error sometimes happens after removing the VNET during fixture_setUp teardown. + # It is a harmless error and does not affect the test results. + # The root cause is a race condition that is fixed by https://github.com/sonic-net/sonic-swss/pull/4499. + # Since the PR is not merged yet, we need to ignore this error for now. + ignore_regex_list = [ + ".*ERR bgp#fpmsyncd:.*onRouteMsg: Invalid VRF name.*" + ] + loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_list) diff --git a/tests/vxlan/test_vnet_decap.py b/tests/vxlan/test_vnet_decap.py index 9098b3c2d3c..0f8859c0fde 100644 --- a/tests/vxlan/test_vnet_decap.py +++ b/tests/vxlan/test_vnet_decap.py @@ -33,16 +33,34 @@ def find_ptf_dest_port(duthost, minigraph_facts, except_interfaces=[]): """ Finds an Ethernet port that is operationally UP and does not appear in except_interfaces and also is not a member of any PortChannel interface that appears in except_interfaces. - Returns the PTF index of that Ethernet port (if such port is found). + Returns the PTF index of that Ethernet port and the port name (if such port is found). """ dut_interfaces = duthost.get_interfaces_status() ptf_indices = minigraph_facts["minigraph_ptf_indices"] except_ports = ecmp_utils.get_ethernet_ports(except_interfaces, minigraph_facts) for intf_name, intf_info in dut_interfaces.items(): if intf_info["oper"] == "up" and intf_name.startswith("Ethernet") and intf_name not in except_ports: - return ptf_indices[intf_name] + return ptf_indices[intf_name], intf_name pytest.skip("No suitable Ethernet port could be found on the DUT for receiving packets from PTF.") - return -1 + return -1, None + + +def get_dest_mac(duthost, tbinfo, minigraph_facts, ingress_intf, router_mac): + """ + Returns the destination MAC that an IP-in-IP packet must carry to be L3-terminated + (and therefore decapsulated) when it ingresses on 'ingress_intf'. + + On t1 the server/downlink ports are routed (L3) interfaces, so the global router MAC + is the termination MAC. On t0/dualtor the server ports are VLAN member ports; routing + for that subnet is done by the VLAN SVI, whose MAC may differ from the router MAC + (on dualtor it is the shared gateway MAC). Using the router MAC on such a port leaves + the frame at L2 and it gets flooded in the VLAN instead of being decapsulated. + """ + if tbinfo["topo"]["type"] == "t0" and ingress_intf is not None: + for vlan_name, vlan_info in minigraph_facts.get("minigraph_vlans", {}).items(): + if ingress_intf in vlan_info.get("members", []): + return duthost.get_dut_iface_mac(vlan_name) + return router_mac @pytest.fixture(scope="module", params=[4, 6], ids=["inner_ipv4", "inner_ipv6"]) @@ -100,9 +118,12 @@ def setup(request, duthosts, rand_one_dut_hostname, tbinfo, inner_ip_version, ou dest_net_prefix=DESTINATION_PREFIX, nexthop_prefix=ENDPOINT_PREFIX, nh_af=outer_ip_version_str) - ptf_port_index = find_ptf_dest_port(duthost, minigraph_facts, except_interfaces=[vnet_interface]) + ptf_port_index, ingress_intf = find_ptf_dest_port(duthost, minigraph_facts, except_interfaces=[vnet_interface]) data = {} # test data data["router_mac"] = router_mac + # On t0/dualtor the ingress port may be a VLAN member, in which case the L3 termination + # (decap) MAC is the VLAN SVI MAC rather than the global router MAC. + data["dest_mac"] = get_dest_mac(duthost, tbinfo, minigraph_facts, ingress_intf, router_mac) data["outer_ip_version"] = outer_ip_version data["inner_ip_version"] = inner_ip_version data["vxlan_src_ip"] = ecmp_utils.get_dut_loopback_address(duthost, minigraph_facts, outer_ip_version_str) @@ -235,6 +256,8 @@ def extract_inner_ip_pkt(outer_pkt, inner_ip_version, outer_ip_version): return packet.IPv6(outer_pkt_bytes[outer_ip_header_size:]) +@pytest.mark.dualtor_active_standby_toggle_to_random_tor +@pytest.mark.dualtor_active_active_setup_standby_on_random_unselected_tor def test_vnet_decap(setup, ptfadapter): """ We send an IP-in-IP packet to the DUT: @@ -246,6 +269,7 @@ def test_vnet_decap(setup, ptfadapter): """ data = setup router_mac = data["router_mac"] + dest_mac = data["dest_mac"] outer_ip_version = data["outer_ip_version"] inner_ip_version = data["inner_ip_version"] vxlan_src_ip = data["vxlan_src_ip"] @@ -256,7 +280,7 @@ def test_vnet_decap(setup, ptfadapter): inner_ip_pkt = get_inner_ip_packet(vnet_dest, inner_ip_version) # Does not have the Ethernet header ptf_mac = ptfadapter.dataplane.get_mac(0, ptf_port_index) - test_pkt = get_outer_packet(ptf_mac, router_mac, vxlan_src_ip, inner_ip_pkt, outer_ip_version) + test_pkt = get_outer_packet(ptf_mac, dest_mac, vxlan_src_ip, inner_ip_pkt, outer_ip_version) expected_pkt = get_expected_vxlan_packet(outer_ip_version, router_mac, vxlan_src_ip, vnet_endpoint, inner_ip_pkt) ptfadapter.dataplane.flush() testutils.send(ptfadapter, ptf_port_index, test_pkt) diff --git a/tests/vxlan/test_vxlan_decap_ttl.py b/tests/vxlan/test_vxlan_decap_ttl.py index baad9f3856e..6333d3a2146 100644 --- a/tests/vxlan/test_vxlan_decap_ttl.py +++ b/tests/vxlan/test_vxlan_decap_ttl.py @@ -127,6 +127,24 @@ def select_ingress_port(duthost): pytest.skip("No oper UP Ethernet interface found on the DUT to be used as ingress port.") +def get_dest_mac(duthost, tbinfo, minigraph_facts, ingress_intf, router_mac): + """ + Returns the destination MAC the outer VxLAN packet must carry to be L3-terminated + (and therefore decapsulated) when it ingresses on 'ingress_intf'. + + On t1 the server/downlink ports are routed (L3) interfaces, so the global router MAC + is the termination MAC. On t0/dualtor the server ports are VLAN member ports; routing + for that subnet is done by the VLAN SVI, whose MAC may differ from the router MAC + (on dualtor it is the shared gateway MAC). Using the router MAC on such a port leaves + the frame at L2 and it gets flooded in the VLAN instead of being decapsulated. + """ + if tbinfo["topo"]["type"] == "t0" and ingress_intf is not None: + for vlan_name, vlan_info in minigraph_facts.get("minigraph_vlans", {}).items(): + if ingress_intf in vlan_info.get("members", []): + return duthost.get_dut_iface_mac(vlan_name) + return router_mac + + def select_egress_ip_and_ports(duthost, minigraph_facts, inner_ip_version, exclude_ports=[]): """ Returns a tuple of (egress_ip, egress_port_list) to be used in tests. @@ -218,6 +236,8 @@ def get_expected_packet_mask(inner_pkt, inner_ip_version): return get_expected_packet_mask_ipv6(inner_pkt) +@pytest.mark.dualtor_active_standby_toggle_to_upper_tor +@pytest.mark.dualtor_active_active_setup_standby_on_lower_tor def test_vxlan_decap_ttl(duthost, tbinfo, ptfadapter, create_vnet, outer_ip_version, inner_ip_version): # noqa F811 """ In this test, the DUT acts as a VNET endpoint and decapulates VxLAN packets sent to it that match @@ -235,9 +255,14 @@ def test_vxlan_decap_ttl(duthost, tbinfo, ptfadapter, create_vnet, outer_ip_vers egress_port_indices = [ptf_indices[port] for port in egress_ports] ptf_src_mac = ptfadapter.dataplane.get_mac(0, ptf_indices[ingress_port]) + # On t0/dualtor the ingress port may be a VLAN member, in which case the outer packet must be + # addressed to the VLAN SVI MAC (the L3 termination MAC) to be decapsulated rather than flooded. + # The inner frame's dst MAC stays the router MAC, which is the configured vxlan_router_mac. + outer_dst_mac = get_dest_mac(duthost, tbinfo, minigraph_facts, ingress_port, router_mac) + inner_pkt = get_inner_packet(dst_mac=router_mac, src_mac=ptf_src_mac, ip_version=inner_ip_version, dst_ip=inner_dst_ip, ttl=2) - outer_pkt = get_outer_packet(eth_dst=router_mac, eth_src=ptf_src_mac, ip_version=outer_ip_version, + outer_pkt = get_outer_packet(eth_dst=outer_dst_mac, eth_src=ptf_src_mac, ip_version=outer_ip_version, ip_dst=vnet_endpoint, inner_pkt=inner_pkt) exp_pkt_mask = get_expected_packet_mask(inner_pkt, inner_ip_version) diff --git a/tests/vxlan/test_vxlan_route_advertisement.py b/tests/vxlan/test_vxlan_route_advertisement.py index e8550c6e6d0..326172d11a9 100644 --- a/tests/vxlan/test_vxlan_route_advertisement.py +++ b/tests/vxlan/test_vxlan_route_advertisement.py @@ -300,7 +300,7 @@ def verify_nighbor_has_routes_scale(self, routes, community=""): result = t2device['host'].run_command(cmd) while len(result['stdout'][0]) == 0 and retry_count > 0: time.sleep(10) - result = self.vxlan_test_setup['t2']['host'].run_command(cmd) + result = t2device['host'].run_command(cmd) retry_count = retry_count - 1 if len(result['stdout'][0]) == 0: py_assert(False, "Routes not propogated to the T2.") diff --git a/tests/wred/test_wred_counters.py b/tests/wred/test_wred_counters.py new file mode 100644 index 00000000000..ac4197aba2c --- /dev/null +++ b/tests/wred/test_wred_counters.py @@ -0,0 +1,536 @@ +import pytest +import logging +import time +import ptf.testutils as testutils +import ptf.packet as packet +import ast + +from tests.common.helpers.assertions import pytest_assert +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from ptf.mask import Mask + + +pytestmark = [pytest.mark.topology("t0", "t1")] + +logger = logging.getLogger(__name__) + +# ASIC DB key patterns +WRED_PROFILE_PATTERN = "ASIC_STATE:SAI_OBJECT_TYPE_WRED:oid:*" +SCHEDULER_PATTERN = "ASIC_STATE:SAI_OBJECT_TYPE_SCHEDULER:oid:*" +SRV6_MY_SID_PATTERN_PREFIX = "ASIC_STATE:SAI_OBJECT_TYPE_MY_SID_ENTRY:" +ROUTE_PATTERN_PREFIX = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:" + +PACKET_COUNT = 100 +QUEUE = 3 +BLOCKING_SCHEDULER = "SCHEDULER_BLOCK_DATA_PLANE" + +# SRv6-related constants +LOCATOR_NAME = "loc_wred" +LOCATOR_PREFIX = "fcbb:bbbb:1::" +LOCATOR_SID_PREFIX = "fcbb:bbbb:1::/48" +NEXTHOP_PREFIX = "fcbb:bbbb:2::/48" +SRV6_OUTER_DIP = "fcbb:bbbb:1:2::" +SRV6_SHIFTED_DIP = "fcbb:bbbb:2::" + +sonic_db_cli = "sonic-db-cli" +namespace = "" + + +@pytest.fixture(scope="module", autouse=True) +def checkpoint(duthost): + create_checkpoint(duthost) + yield + try: + rollback_or_reload(duthost) + finally: + delete_checkpoint(duthost) + + +@pytest.fixture(scope="module", autouse=True) +def set_namespace(duthost, enum_frontend_asic_index): + global namespace + if duthost.is_multi_asic: + namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index) + else: + namespace = "" + + +def get_namespace_option(): + global namespace + if namespace: + return f"-n {namespace}" + else: + return "" + + +@pytest.fixture(scope="module", autouse=True) +def set_sonic_db_cli(set_namespace): # noqa F811 + global sonic_db_cli + sonic_db_cli = f"sonic-db-cli {get_namespace_option()}" + + +# autouse=True so that we get the counts before any WRED profile or scheduler is created. +@pytest.fixture(scope="module", autouse=True) +def old_asic_db_counts(duthost): + return { + WRED_PROFILE_PATTERN: count_keys(duthost, "ASIC_DB", WRED_PROFILE_PATTERN), + SCHEDULER_PATTERN: count_keys(duthost, "ASIC_DB", SCHEDULER_PATTERN) + } + + +def count_keys(duthost, db, pattern): + result = duthost.shell(f"{sonic_db_cli} {db} KEYS '{pattern}'")["stdout"].strip() + if not result: + return 0 + return len(result.splitlines()) + + +def pattern_exists(duthost, db, key): + return count_keys(duthost, db, key) > 0 + + +@pytest.fixture(scope="module") +def enable_wred_counters(duthost): + logger.info("Enabling WRED counters on the DUT.") + duthost.shell("counterpoll wredqueue enable") + + +@pytest.fixture(scope="module") +def create_blocking_scheduler(duthost): + logger.info(f"Creating the blocking scheduler '{BLOCKING_SCHEDULER}' on the DUT.") + cmd = f"{sonic_db_cli} CONFIG_DB HSET 'SCHEDULER|{BLOCKING_SCHEDULER}' 'type' 'DWRR' 'weight' '15' \ + 'pir' '1' 'cir' '1'" + if duthost.facts["asic_type"] == "broadcom": + cmd += " 'meter_type' 'packets'" + duthost.shell(cmd) + # No need to wait here since we will wait in the "setup" fixture. + + +@pytest.fixture(scope="module") +def is_srv6_supported(duthost): + ret = False + result = duthost.shell("crm show resources srv6-my-sid-entry", module_ignore_errors=True) + # The output of the above command looks like this if SRv6 is supported: + # Resource Name Used Count Available Count + # ----------------- ------------ ----------------- + # srv6_my_sid_entry 0 128 + if result["rc"] == 0: + for line in result["stdout"].splitlines(): + fields = line.split() + if len(fields) >= 3 and fields[0] == "srv6_my_sid_entry": + try: + ret = int(fields[2]) > 0 + except ValueError as e: + logger.error(f"Failed to parse SRv6 resource count: {e}") + break + logger.info(f"SRv6 is {'supported' if ret else 'NOT supported'} on this DUT.") + return ret + + +@pytest.fixture(scope="module") +def configure_srv6(duthost, is_srv6_supported): + if is_srv6_supported: + logger.info("Configuring SRv6 on the DUT...") + duthost.command( + f"{sonic_db_cli} CONFIG_DB HSET 'SRV6_MY_LOCATORS|{LOCATOR_NAME}' \ + 'prefix' '{LOCATOR_PREFIX}' 'func_len' '0'" + ) + duthost.command( + f"{sonic_db_cli} CONFIG_DB HSET 'SRV6_MY_SIDS|{LOCATOR_NAME}|{LOCATOR_SID_PREFIX}' \ + 'action' 'uN' 'decap_dscp_mode' 'pipe'" + ) + # Static route to NEXTHOP_PREFIX will be added later since it depends on the selected egress interface. + # No need to wait here since we will wait in the "setup" fixture + + +# This fixture is not in module scope because we don't want to teardown other fixtures in module scope when +# we switch from "drop" to "ecn". +@pytest.fixture(params=["drop", "ecn"]) +def create_wred_profile(duthost, request): + profile_name = f"TEST_WRED_{request.param.upper()}" + if request.param == "drop": + ecn = "ecn_none" + else: + ecn = "ecn_green" + logger.info(f"Creating the WRED profile '{profile_name}' with action '{request.param}' on the DUT.") + duthost.shell(f"{sonic_db_cli} CONFIG_DB HSET 'WRED_PROFILE|{profile_name}' 'wred_green_enable' 'true'\ + 'ecn' '{ecn}' 'green_drop_probability' '100' 'green_max_threshold' '1'\ + 'green_min_threshold' '1'") + # No need to wait here since we will wait in the "setup" fixture after applying the profile to the queue. + return (request.param, profile_name) + + +def select_egress_interface_ipv4(duthost, portchannel_info): + ip_interfaces = duthost.show_ip_interface()["ansible_facts"]["ip_interfaces"] + for intf, info in ip_interfaces.items(): + if (intf.startswith("Ethernet") or intf.startswith("PortChannel")) and info["oper_state"].lower() == "up": + neigh_ip = info.get("peer_ipv4", "") + if not neigh_ip or neigh_ip.lower() == "n/a": + continue + + if intf.startswith("PortChannel"): + members = portchannel_info[intf]["members"] + else: + members = [intf] + logger.info(f"Selected egress packet's dest IPv4 '{neigh_ip}' and egress interface '{intf}'.") + return (neigh_ip, members) + pytest.skip("No suitable egress interface found on the DUT.") + + +def select_egress_interface_ipv6(duthost, portchannel_info): + ip_interfaces = duthost.show_ipv6_interfaces() + for intf, info in ip_interfaces.items(): + if (intf.startswith("Ethernet") or intf.startswith("PortChannel")) and info["oper"].lower() == "up": + neigh_ip = info.get("neighbor ip", "") + if not neigh_ip or neigh_ip.lower() == "n/a": + continue + + if intf.startswith("PortChannel"): + members = portchannel_info[intf]["members"] + else: + members = [intf] + logger.info(f"Selected egress packet's dest IPv6 '{neigh_ip}' and egress interface '{intf}'.") + return (neigh_ip, members) + pytest.skip("No suitable egress interface found on the DUT.") + + +def select_egress_interface(duthost, minigraph_facts, ipv4=True): + """ + Find an Ethernet or a PortChannel interface that is oper UP and has a neighbor IP. + Traffic sent to DUT will go out from interface. + """ + portchannel_info = minigraph_facts["minigraph_portchannels"] + if ipv4: + return select_egress_interface_ipv4(duthost, portchannel_info) + else: + return select_egress_interface_ipv6(duthost, portchannel_info) + + +def select_ingress_port(duthost, exclude_ports=[]): + """ + Returns the name of an oper UP Ethernet interface that is not in the exclude_ports list. + PTF will send traffic to this interface. + """ + interfaces_status = duthost.show_interface(command="status")["ansible_facts"]["int_status"] + for intf, info in interfaces_status.items(): + if info["oper_state"].lower() == "up" and intf.startswith("Ethernet") and intf not in exclude_ports: + logger.info(f"Selected '{intf}' as ingress port.") + return intf + pytest.skip("No suitable ingress port found on the DUT.") + + +def find_qos_mapping_table_name(duthost, egress_ports, qos_mapping): + qos_table_name = "" + for port in egress_ports: + table = duthost.shell(f"{sonic_db_cli} CONFIG_DB HGET 'PORT_QOS_MAP|{port}' '{qos_mapping}'")["stdout"].strip() + pytest_assert(table, f"{qos_mapping} is not defined for port {port}.") + if qos_table_name and qos_table_name != table: + pytest.skip(f"{qos_mapping} is not the same for all egress ports {egress_ports}.") + qos_table_name = table + return qos_table_name + + +def find_reverse_qos_mapping(duthost, egress_ports, qos_mapping, value_to_find): + qos_table = find_qos_mapping_table_name(duthost, egress_ports, qos_mapping) + qos_map_str = \ + duthost.shell(f"{sonic_db_cli} CONFIG_DB HGETALL '{qos_mapping.upper()}|{qos_table}'")["stdout"].strip() + qos_map = ast.literal_eval(qos_map_str) + for key, value in qos_map.items(): + if int(value) == value_to_find: + return int(key) + pytest.skip(f"Could not find a key mapped to value {value_to_find} in {qos_mapping.upper()}|{qos_table}.") + + +def find_tc_for_queue(duthost, egress_ports, queue): + tc = find_reverse_qos_mapping(duthost, egress_ports, "tc_to_queue_map", queue) + logger.info(f"The traffic class '{tc}' is mapped to queue '{queue}' for egress ports {egress_ports}.") + return tc + + +def find_dscp_for_queue(duthost, egress_ports, queue): + tc = find_tc_for_queue(duthost, egress_ports, queue) + dscp = find_reverse_qos_mapping(duthost, egress_ports, "dscp_to_tc_map", tc) + logger.info(f"The DSCP value '{dscp}' is mapped to traffic class '{tc}' for egress ports {egress_ports}.") + return dscp + + +def check_asic_db_counts(duthost, old_counts, new_wred_profile): + new_wred_profile_count = count_keys(duthost, "ASIC_DB", WRED_PROFILE_PATTERN) + # We create two WRED profiles in total: TEST_WRED_ECN and TEST_WRED_DROP + pytest_assert(new_wred_profile_count >= old_counts[WRED_PROFILE_PATTERN] + 1 and + new_wred_profile_count <= old_counts[WRED_PROFILE_PATTERN] + 2, + f"WRED profile {new_wred_profile} was not added to ASIC DB.") + new_scheduler_count = count_keys(duthost, "ASIC_DB", SCHEDULER_PATTERN) + pytest_assert(new_scheduler_count == old_counts[SCHEDULER_PATTERN] + 1, + f"Scheduler {BLOCKING_SCHEDULER} was not added to ASIC DB.") + + +@pytest.fixture(params=[ + ("no-SRv6", "ipv4"), + ("no-SRv6", "ipv6"), + ("SRv6", "ipv4"), + ("SRv6", "ipv6"), + ("SRv6-with-SRH", "ipv4"), + ("SRv6-with-SRH", "ipv6") +], ids=[ + "ipv4", + "ipv6", + "SRv6_inner_ipv4", + "SRv6_inner_ipv6", + "SRv6_with_SRH_inner_ipv4", + "SRv6_with_SRH_inner_ipv6" +]) +def setup(duthost, tbinfo, request, enable_wred_counters, create_blocking_scheduler, create_wred_profile, # noqa F811 + configure_srv6, is_srv6_supported, old_asic_db_counts): # noqa F811 + minigraph_facts = duthost.get_extended_minigraph_facts(tbinfo) + ptf_indices = minigraph_facts["minigraph_ptf_indices"] + is_srv6_test = request.param[0].startswith("SRv6") + + if not is_srv6_supported and is_srv6_test: + pytest.skip("SRv6 is not supported on this platform.") + test_params = {} + # test_params["inner_ip_version"] = Inner IP version for SRv6 packets or IP version for regular packets + test_params["inner_ip_version"] = request.param[1] + # test_params["ip_version"] = "ipv6" for SRv6 packets or IP version for regular packets + test_params["ip_version"] = "ipv6" if is_srv6_test else request.param[1] + test_params["srv6"] = is_srv6_test + test_params["with_srh"] = request.param[0].endswith("with-SRH") + neigh_ip, egress_ports = select_egress_interface(duthost, minigraph_facts, + ipv4=(test_params["ip_version"] == "ipv4")) + test_params["neigh_ip"] = neigh_ip + test_params["egress_ports"] = {port: ptf_indices[port] for port in egress_ports} + ingress_port = select_ingress_port(duthost, exclude_ports=egress_ports) + test_params["ingress_port_index"] = ptf_indices[ingress_port] + policy, wred_profile_name = create_wred_profile + test_params["policy"] = policy + test_params["wred_profile_name"] = wred_profile_name + test_params["dscp"] = find_dscp_for_queue(duthost, egress_ports, QUEUE) + + test_params["prev_scheduler"] = {} + for port in egress_ports: + logger.info(f"Setting the WRED profile of {port}|{QUEUE} to '{wred_profile_name}'.") + duthost.shell(f"{sonic_db_cli} CONFIG_DB HSET 'QUEUE|{port}|{QUEUE}' 'wred_profile' '{wred_profile_name}'") + test_params["prev_scheduler"][port] = \ + duthost.shell(f"{sonic_db_cli} CONFIG_DB HGET 'QUEUE|{port}|{QUEUE}' 'scheduler'")["stdout"].strip() + logger.info(f"The original scheduler of {port}|{QUEUE} is '{test_params['prev_scheduler'][port]}'.") + logger.info(f"Setting the scheduler of {port}|{QUEUE} to '{BLOCKING_SCHEDULER}'.") + duthost.shell(f"{sonic_db_cli} CONFIG_DB HSET 'QUEUE|{port}|{QUEUE}' 'scheduler' '{BLOCKING_SCHEDULER}'") + + # Add a static route to NEXTHOP_PREFIX via the selected egress interface if testing with SRv6 packets. + if is_srv6_test: + logger.info(f"Adding a static route to {NEXTHOP_PREFIX} via {neigh_ip}.") + duthost.shell(f"sudo config route add prefix {NEXTHOP_PREFIX} nexthop {neigh_ip}") + + logger.info("Waiting 10 seconds for the configuration to take effect.") + time.sleep(10) # Wait for the configuration to take effect + + # Verifying that the WRED profile, static route, blocking scheduler, and SRv6 configuration are applied to ASIC DB. + check_asic_db_counts(duthost, old_asic_db_counts, wred_profile_name) + if is_srv6_test: + srv6_my_sid_pattern = f'{SRV6_MY_SID_PATTERN_PREFIX}*\\"sid\\":\\"{LOCATOR_PREFIX}\\"*' + pytest_assert(pattern_exists(duthost, "ASIC_DB", srv6_my_sid_pattern), + f"SRv6 MY_SID entry for {LOCATOR_PREFIX} was not added to ASIC DB.") + route_pattern = f'{ROUTE_PATTERN_PREFIX}*\\"dest\\":\\"{NEXTHOP_PREFIX}\\"*' + pytest_assert(pattern_exists(duthost, "ASIC_DB", route_pattern), + f"Route entry for {NEXTHOP_PREFIX} was not added to ASIC DB.") + + logger.info(f"Test parameters: {test_params}") + return test_params + + +def get_srv6_test_packet(dest_mac, inner_ip_version, ecn, dscp, with_srh): + if inner_ip_version == "ipv4": + inner_eth = testutils.simple_udp_packet() + inner_ip = inner_eth["IP"] + else: + inner_eth = testutils.simple_udpv6_packet() + inner_ip = inner_eth["IPv6"] + + if with_srh: + pkt = testutils.simple_ipv6_sr_packet( + eth_dst=dest_mac, + ipv6_dst=SRV6_OUTER_DIP, + ipv6_tc=testutils.ip_make_tos(0, ecn, dscp), + srh_seg_left=1, + srh_nh=4 if inner_ip_version == "ipv4" else 41, + inner_frame=inner_ip + ) + else: + pkt = testutils.simple_ipv6ip_packet( + eth_dst=dest_mac, + ipv6_dst=SRV6_OUTER_DIP, + ipv6_ecn=ecn, + ipv6_dscp=dscp, + inner_frame=inner_ip + ) + return pkt + + +def get_test_packet(dest_mac, ip_version, neigh_ip, ect_enabled, dscp, srv6, with_srh): + """ + For normal packets, ip_version is the IP version of the packet. + For SRv6 packets, ip_version is the IP version of the inner packet. + """ + ecn = 0b10 if ect_enabled else 0b00 + if srv6: + return get_srv6_test_packet(dest_mac, ip_version, ecn, dscp, with_srh) + if ip_version == "ipv4": + pkt = testutils.simple_udp_packet( + eth_dst=dest_mac, + ip_dst=neigh_ip, + ip_ecn=ecn, + ip_dscp=dscp, + ) + else: + pkt = testutils.simple_udpv6_packet( + eth_dst=dest_mac, + ipv6_dst=neigh_ip, + ipv6_ecn=ecn, + ipv6_dscp=dscp, + ) + return pkt + + +def get_congestion_packet(dest_mac, ip_version, dest_ip, ect_enabled, dscp): + ecn = 0b10 if ect_enabled else 0b00 + if ip_version == "ipv4": + pkt = testutils.simple_tcp_packet( + eth_dst=dest_mac, + ip_dst=dest_ip, + ip_ecn=ecn, + ip_dscp=dscp, + ) + else: + pkt = testutils.simple_tcpv6_packet( + eth_dst=dest_mac, + ipv6_dst=dest_ip, + ipv6_ecn=ecn, + ipv6_dscp=dscp, + ) + return pkt + + +def get_expected_packet_mask_ipv4(pkt, expected_dest_ip): + exp_pkt = pkt.copy() + exp_pkt["IP"].ttl -= 1 + exp_pkt["IP"].tos |= 0b11 # Set ECN bits to '11' (CE) + exp_pkt["IP"].dst = expected_dest_ip + exp_pkt_mask = Mask(exp_pkt) + exp_pkt_mask.set_do_not_care_packet(packet.Ether, "dst") + exp_pkt_mask.set_do_not_care_packet(packet.Ether, "src") + exp_pkt_mask.set_do_not_care_packet(packet.IP, "ihl") + exp_pkt_mask.set_do_not_care_packet(packet.IP, "id") + exp_pkt_mask.set_do_not_care_packet(packet.IP, "flags") + exp_pkt_mask.set_do_not_care_packet(packet.IP, "chksum") + exp_pkt_mask.set_do_not_care_packet(packet.UDP, "chksum") + return exp_pkt_mask + + +def get_expected_packet_mask_ipv6(pkt, expected_dest_ip): + exp_pkt = pkt.copy() + exp_pkt["IPv6"].hlim -= 1 + exp_pkt["IPv6"].tc |= 0b11 # Set ECN bits to '11' (CE) + exp_pkt["IPv6"].dst = expected_dest_ip + exp_pkt_mask = Mask(exp_pkt) + exp_pkt_mask.set_do_not_care_packet(packet.Ether, "dst") + exp_pkt_mask.set_do_not_care_packet(packet.Ether, "src") + exp_pkt_mask.set_do_not_care_packet(packet.IPv6, "fl") + exp_pkt_mask.set_do_not_care_packet(packet.UDP, "chksum") + return exp_pkt_mask + + +def get_expected_packet_mask(pkt, ip_version, expected_dest_ip): + if ip_version == "ipv4": + return get_expected_packet_mask_ipv4(pkt, expected_dest_ip) + else: + return get_expected_packet_mask_ipv6(pkt, expected_dest_ip) + + +def get_wred_counters(duthost, port, queue): + wred_counters_str = \ + duthost.shell(f"show queue wredcounters {get_namespace_option()} --json {port}")["stdout"].strip() + wred_counters = ast.literal_eval(wred_counters_str) + return wred_counters[port][f"UC{queue}"] + + +def check_wred_counters(duthost, egress_ports, expect_drop, expect_zero=False): + action = "drop" if expect_drop else "ECN" + count = 0 + if expect_drop: + counter_key = "wreddroppacket" + else: + counter_key = "ecnmarkedpacket" + for port in egress_ports: + wred_counters = get_wred_counters(duthost, port, QUEUE) + queue_count_str = wred_counters.get(counter_key, "N/A") + logger.info(f"WRED {action} counter for {port}|{QUEUE} is {queue_count_str}.") + pytest_assert(queue_count_str.isdigit(), + f"Could not get the WRED {action} counter for queue {port}|{QUEUE}.") + count += int(queue_count_str) + if expect_zero: + pytest_assert(count == 0, + f"Sum of WRED {action} counters ({count}) is not zero after clearing counters.") + else: + pytest_assert(count >= PACKET_COUNT, + f"Sum of WRED {action} counters ({count}) is less than {PACKET_COUNT}.") + logger.info(f"Sum of WRED {action} counters across egress ports {egress_ports} is {count}.") + + +def clear_queue_wred_counters(duthost, egress_ports, expect_drop): + logger.info("Clearing WRED counters on the DUT.") + duthost.shell("sonic-clear queue wredcounters") + check_wred_counters(duthost, egress_ports, expect_drop, expect_zero=True) + + +def create_congestion(ptfadapter, router_mac, ip_version, neigh_ip, dscp, ingress_port_index, egress_port_count): + logger.info("Creating congestion on egress queues.") + # Using TCP for congestion packets so that we can distinguish them from the test packets. + pkt = get_congestion_packet(router_mac, ip_version, neigh_ip, ect_enabled=True, dscp=dscp) + logger.info(f"Congestion packet: {pkt}") + # The goal is to put at least 15KB of data into each egress queue. We send 20KB to each queue to account for + # possible traffic imbalance among LAG members. + pkt_len = len(pkt) + num_packets = ((20000 + pkt_len - 1) // pkt_len) * egress_port_count + logger.info(f"Sending {num_packets} congestion packets to ingress port {ingress_port_index}.") + testutils.send(ptfadapter, ingress_port_index, pkt, count=num_packets) + + +def restore_original_schedulers(duthost, prev_schedulers): + logger.info("Restoring original schedulers for all egress queues.") + for port, scheduler in prev_schedulers.items(): + if scheduler: + logger.info(f"Restoring scheduler of {port}|{QUEUE} to '{scheduler}'.") + duthost.shell(f"{sonic_db_cli} CONFIG_DB HSET 'QUEUE|{port}|{QUEUE}' 'scheduler' '{scheduler}'") + logger.info("Waiting 10 seconds for the configuration to take effect.") + time.sleep(10) # Wait for the configuration to take effect + + +@pytest.mark.parametrize("ect_enabled", [False, True], ids=["ect_disabled", "ect_enabled"]) +def test_wred_counters(duthost, ptfadapter, setup, ect_enabled): + test_params = setup + expect_ecn = ect_enabled and test_params["policy"] == "ecn" + router_mac = duthost.facts["router_mac"] + pkt = get_test_packet(router_mac, test_params["inner_ip_version"], test_params["neigh_ip"], + ect_enabled, test_params["dscp"], test_params["srv6"], test_params["with_srh"]) + logger.info(f"Test packet: {pkt}") + expected_dest_ip = SRV6_SHIFTED_DIP if test_params["srv6"] else test_params["neigh_ip"] + exp_pkt_mask = get_expected_packet_mask(pkt, test_params["ip_version"], expected_dest_ip) + clear_queue_wred_counters(duthost, list(test_params["egress_ports"].keys()), expect_drop=not expect_ecn) + + create_congestion(ptfadapter, router_mac, test_params["ip_version"], test_params["neigh_ip"], + test_params["dscp"], test_params["ingress_port_index"], len(test_params["egress_ports"])) + + ptfadapter.dataplane.flush() + logger.info(f"Sending {PACKET_COUNT} test packets to ingress port {test_params['ingress_port_index']}.") + testutils.send(ptfadapter, test_params["ingress_port_index"], pkt, count=PACKET_COUNT) + + # Restoring original schedulers so that egress queues are unblocked. This step is necessary since otherwise, + # all test packets could remain in the queues and we cannot capture and verify egress packets. + restore_original_schedulers(duthost, test_params["prev_scheduler"]) + if expect_ecn: + _, received_pkt = testutils.verify_packet_any_port(ptfadapter, exp_pkt_mask, + ports=list(test_params["egress_ports"].values())) + logger.info(f"Received packet: {packet.Ether(received_pkt)}") + else: + testutils.verify_no_packet_any(ptfadapter, exp_pkt_mask, ports=list(test_params["egress_ports"].values())) + check_wred_counters(duthost, list(test_params["egress_ports"].keys()), expect_drop=not expect_ecn)