From 9f0d513a595770dc3c5b7b63b98be1ad9df57c7b Mon Sep 17 00:00:00 2001 From: Muneer CH Date: Wed, 10 Jun 2026 05:45:35 +0000 Subject: [PATCH 1/5] scripts for link dampening feature --- .../README_LINK_EVENT_DAMPING.md | 1403 +++++++++++++++++ tests/link_dampening/conftest.py | 93 ++ .../link_event_damping_utils.py | 712 +++++++++ .../link_dampening/test_link_event_damping.py | 1403 +++++++++++++++++ 4 files changed, 3611 insertions(+) create mode 100644 tests/link_dampening/README_LINK_EVENT_DAMPING.md create mode 100644 tests/link_dampening/conftest.py create mode 100644 tests/link_dampening/link_event_damping_utils.py create mode 100644 tests/link_dampening/test_link_event_damping.py diff --git a/tests/link_dampening/README_LINK_EVENT_DAMPING.md b/tests/link_dampening/README_LINK_EVENT_DAMPING.md new file mode 100644 index 00000000000..bd351952e8e --- /dev/null +++ b/tests/link_dampening/README_LINK_EVENT_DAMPING.md @@ -0,0 +1,1403 @@ +# Link Event Damping Test Suite - README + +## Overview + +This document provides comprehensive documentation for the Link Event Damping test suite implemented in SONiC. The test suite validates the correctness, robustness, and compliance of the Link Event Damping feature with the High-Level Design (HLD). + +**Test Suite Location**: `tests/link_event_damping/` + +**Files**: +- `test_link_event_damping.py` - Main test file with 53 test cases +- `link_event_damping_utils.py` - Utility functions and helpers +- `conftest.py` - Pytest fixtures and configuration + +**Topologies Supported**: T0, T1 + +--- + +## Prerequisites + +### Hardware Requirements +- SONiC-supported switch with multiple Ethernet ports +- Fanout switch(es) for generating deterministic link flaps +- Access to front-facing interfaces (at least 2 for basic tests, 5+ for comprehensive tests) + +### Software Requirements +- SONiC image with Link Event Damping support +- Access to: + - SONiC CLI (config and show commands) + - Redis databases (CONFIG_DB, APP_DB, STATE_DB) + - Docker CLI for container restart tests + - System logs + +### Test Environment Setup +```bash +# Install sonic-mgmt test dependencies +pip install -r requirements.txt + +# Verify test environment +pytest --version +python3 -m py_compile tests/link_event_damping/test_link_event_damping.py +``` + +--- + +## Quick Start + +### Running All Tests +```bash +# Run all link event damping tests +pytest tests/link_event_damping/test_link_event_damping.py -v + +# Run with detailed logging +pytest tests/link_event_damping/test_link_event_damping.py -v -s + +# Run specific test class +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingBasics -v +``` + +### Running Specific Test Cases +```bash +# TC01.1 - Normal Link Flap Event Propagation +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingBasics::test_tc01_1_normal_link_flap_event_propagation -v + +# TC02.1 - Basic Link Damping Configuration +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingConfiguration::test_tc02_1_basic_link_damping_configuration -v + +# All TC03 (Unsupported Configuration) tests +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingUnsupported -v +``` + +### Test Execution with Topology +```bash +# Run tests on specific topology +pytest tests/link_event_damping/test_link_event_damping.py -v -k "topology" +``` + +--- + +## Test Cases Documentation + +### Test Configuration Parameters + +**Standard Damping Configuration (DAMPING_CONFIG_PARAMS)**: +``` +suppress_threshold: 1600 # Penalty threshold to enter suppressed state +reuse_threshold: 1200 # Penalty threshold to exit suppressed state +decay_half_life: 15 seconds # Time for penalty to decay by half +max_suppress_time: 30 seconds # Maximum suppression duration +flap_penalty: 500 # Penalty per link down event +``` + +--- + +## TC01: Normal Link Flap Event Propagation + +**Objective**: Verify that link up/down events propagate normally when damping is inactive. + +### TC01.1 - Normal Link Flap Event Propagation + +**Procedure**: +1. Get a test interface from the DUT +2. Ensure damping is **disabled** on the interface +3. Record initial interface physical state +4. Generate 5 link flaps via fanout switch with 1 second interval +5. Monitor interface operational state +6. Retrieve pre-damping link transition counters +7. Verify operational state matches physical state + +**Expected Results**: +- ✓ Damping is successfully disabled +- ✓ All physical link changes are propagated +- ✓ Pre-damping link transitions counter > 0 +- ✓ Operational state matches physical state +- ✓ No events are suppressed + +**Duration**: ~10 seconds + +--- + +### TC01.2 - Multiple Sequential Flaps + +**Procedure**: +1. Get a test interface +2. Disable damping on the interface +3. Clear all link damping statistics +4. Generate 10 sequential flaps with 0.5 second interval between flaps +5. Retrieve counters: pre-damping DOWN events and pre-damping UP events +6. Verify counter values + +**Expected Results**: +- ✓ DOWN events counter ≥ (10/2) = 5 +- ✓ UP events counter ≥ (10/2) = 5 +- ✓ All flaps are recorded in statistics +- ✓ Counters reflect actual number of events + +**Duration**: ~10 seconds + +--- + +### TC01.3 - Simultaneous Flaps on Multiple Ports + +**Procedure**: +1. Get 3 test interfaces from the DUT +2. Disable damping on all 3 interfaces +3. Clear statistics +4. Generate 3 link flaps with 0.5 second interval on all interfaces simultaneously +5. Verify each interface has recorded events independently + +**Expected Results**: +- ✓ All 3 interfaces report pre-damping transitions > 0 +- ✓ Each interface's counters are independent +- ✓ No counter cross-talk between interfaces +- ✓ Concurrent flaps handled correctly + +**Duration**: ~5 seconds per interface + +--- + +## TC02: Valid Damping Configuration + +**Objective**: Verify that all damping parameters are configurable and persistent. + +### TC02.1 - Basic Link Damping Configuration + +**Procedure**: +1. Select a test interface +2. Configure damping with all parameters: + - suppress_threshold: 1600 + - reuse_threshold: 1200 + - decay_half_life: 15 seconds + - max_suppress_time: 30 seconds + - flap_penalty: 500 +3. Verify configuration was applied + +**Expected Results**: +- ✓ Configuration command accepted +- ✓ No error messages in output +- ✓ Configuration parameters appear in CONFIG_DB +- ✓ Interface damping is now active + +**Duration**: ~2 seconds + +--- + +### TC02.2 - CONFIG_DB Persistence + +**Procedure**: +1. Configure damping on a test interface with standard parameters +2. Query CONFIG_DB via `redis-cli -n 4 HGETALL 'LINK_DAMPING|'` +3. Verify all configuration parameters are present +4. Verify values match configured parameters + +**Expected Results**: +- ✓ CONFIG_DB entry exists for interface +- ✓ All parameters are queryable +- ✓ Values match configuration +- ✓ persist flag is set (if applicable) + +**Duration**: ~2 seconds + +--- + +### TC02.3 - Redis Persistence + +**Procedure**: +1. Configure damping on test interface +2. Query multiple Redis databases: + - CONFIG_DB (index 4) + - APP_DB (index 0) + - STATE_DB (index 1) +3. Verify entries exist in appropriate databases +4. Validate configuration consistency across databases + +**Expected Results**: +- ✓ CONFIG_DB has configuration entry +- ✓ APP_DB has operational entry (if applicable) +- ✓ STATE_DB has state entry (if applicable) +- ✓ Data is consistent across all databases + +**Duration**: ~2 seconds + +--- + +### TC02.4 - Multiple Configuration Profiles + +**Procedure**: +1. Get 2 test interfaces +2. Apply different configurations to each: + - Interface 1: suppress_threshold=1600, max_suppress_time=30 + - Interface 2: suppress_threshold=800, max_suppress_time=20 +3. Query CONFIG_DB for both interfaces +4. Verify configurations are independent + +**Expected Results**: +- ✓ Interface 1 has configuration with threshold=1600 +- ✓ Interface 2 has configuration with threshold=800 +- ✓ No cross-interference between configurations +- ✓ Both interfaces function independently + +**Duration**: ~3 seconds + +--- + +### TC02.5 - Individual Parameter Validation + +**Procedure**: +1. Select a test interface +2. Configure each parameter individually: + - suppress_threshold: 2000 + - reuse_threshold: 1500 + - decay_half_life: 20 + - max_suppress_time: 45 + - flap_penalty: 600 +3. After each parameter, verify in CONFIG_DB +4. Repeat for each parameter + +**Expected Results**: +- ✓ Each parameter is individually configurable +- ✓ Parameter changes don't affect others +- ✓ Values persist in CONFIG_DB +- ✓ No validation errors + +**Duration**: ~5 seconds + +--- + +### TC02.6 - Configuration Synchronization + +**Procedure**: +1. Configure damping on test interface +2. Verify in CONFIG_DB (redis-cli -n 4) +3. Verify in APP_DB (redis-cli -n 0) +4. Verify in operational state (show commands if available) +5. Check all layers have synchronized data + +**Expected Results**: +- ✓ CONFIG_DB has configuration +- ✓ APP_DB has synchronized copy +- ✓ Operational state reflects configuration +- ✓ All layers in sync within 1 second + +**Duration**: ~2 seconds + +--- + +## TC03: Unsupported Configuration Handling + +**Objective**: Ensure unsupported damping configurations are handled safely. + +### TC03.1 - Decay Exceeds Max Suppress Time + +**Procedure**: +1. Select test interface +2. Configure with unsupported parameters: + - decay_half_life: 45 seconds (> max_suppress_time of 30) + - max_suppress_time: 30 seconds +3. Generate 5 link flaps with 0.5 second interval +4. Retrieve statistics: + - pre_damping_link_transitions + - post_damping_propagated_transitions +5. Compare event counts + +**Expected Results**: +- ✓ Configuration is accepted (or gracefully rejected) +- ✓ Damping is **disabled** (unsupported config) +- ✓ All events are propagated +- ✓ post_damping_propagated ≈ pre_damping_link_transitions +- ✓ No suppression occurs + +**Duration**: ~5 seconds + +--- + +### TC03.2 - Zero Flap Penalty + +**Procedure**: +1. Configure with flap_penalty: 0 +2. Generate 5 link flaps +3. Verify configuration is accepted +4. Check if damping is disabled (penalty accumulation impossible) + +**Expected Results**: +- ✓ Configuration is accepted +- ✓ No validation errors +- ✓ Damping functionality degrades gracefully (if applicable) + +**Duration**: ~2 seconds + +--- + +### TC03.3 - Suppress Less Than Reuse + +**Procedure**: +1. Configure with invalid parameters: + - suppress_threshold: 800 (less than reuse) + - reuse_threshold: 1000 +2. Attempt to apply configuration +3. Check for validation errors + +**Expected Results**: +- ✓ Configuration either rejected OR handled gracefully +- ✓ Error message logged (if rejected) +- ✓ System remains stable + +**Duration**: ~2 seconds + +--- + +### TC03.4 - Zero Reuse Threshold + +**Procedure**: +1. Configure with reuse_threshold: 0 +2. Generate link flaps +3. Verify configuration is accepted +4. Check suppression behavior + +**Expected Results**: +- ✓ Configuration is accepted +- ✓ System handles gracefully +- ✓ No crashes or errors + +**Duration**: ~2 seconds + +--- + +### TC03.6 - Zero Max Suppress Time + +**Procedure**: +1. Configure with max_suppress_time: 0 +2. Generate flaps to trigger suppression +3. Monitor suppression duration +4. Check system behavior + +**Expected Results**: +- ✓ Configuration handled gracefully +- ✓ No infinite suppression +- ✓ System remains stable + +**Duration**: ~2 seconds + +--- + +### TC03.9 - Error Logging + +**Procedure**: +1. Apply invalid configuration +2. Check system logs: + - `/var/log/syslog` + - `/var/log/swss.log` + - `/var/log/syncd.log` +3. Verify appropriate error messages are logged + +**Expected Results**: +- ✓ Error messages logged for invalid configs +- ✓ Messages are clear and actionable +- ✓ No spurious warnings for valid configs + +**Duration**: ~2 seconds + +--- + +## TC04: Multiple Ports with Mixed Damping Configuration + +**Objective**: Verify independent behavior across multiple interfaces. + +### TC04.1 - Basic Mixed Configuration + +**Procedure**: +1. Get 2 test interfaces +2. Enable damping on interface 1 with standard parameters +3. Disable damping on interface 2 +4. Verify both configurations independently + +**Expected Results**: +- ✓ Interface 1 has damping enabled (CONFIG_DB verified) +- ✓ Interface 2 has damping disabled +- ✓ No cross-interference +- ✓ Independent operation confirmed + +**Duration**: ~3 seconds + +--- + +### TC04.2 - Simultaneous Flaps (Damped vs Undamped) + +**Procedure**: +1. Get 2 interfaces (damped and undamped) +2. Configure as per TC04.1 +3. Clear statistics on both +4. Generate 10 identical flaps simultaneously on both +5. Retrieve post-damping propagated transition counters + +**Expected Results**: +- ✓ Undamped interface propagated_transitions ≈ 10 +- ✓ Damped interface propagated_transitions < 10 (if suppression triggered) +- ✓ Undamped >= damped propagated events +- ✓ Independent damping verified + +**Duration**: ~8 seconds + +--- + +### TC04.3 - Different Damping Profiles + +**Procedure**: +1. Get 2 interfaces +2. Apply Profile A to interface 1: + - suppress_threshold: 1600 + - max_suppress_time: 30 +3. Apply Profile B to interface 2: + - suppress_threshold: 800 + - max_suppress_time: 20 +4. Verify configurations in CONFIG_DB + +**Expected Results**: +- ✓ Interface 1 has suppress_threshold=1600 +- ✓ Interface 2 has suppress_threshold=800 +- ✓ max_suppress_time values differ +- ✓ Different suppression behaviors expected + +**Duration**: ~2 seconds + +--- + +### TC04.4 - Port Independence + +**Procedure**: +1. Get 3 interfaces +2. Enable damping on interface 1 +3. Generate 10 flaps on interface 1 (may trigger suppression) +4. Check operational states of interfaces 2 and 3 +5. Verify interfaces 2 and 3 are unaffected + +**Expected Results**: +- ✓ Interface 2 and 3 operational states normal +- ✓ No flap-induced transitions on interfaces 2 and 3 +- ✓ Complete port independence +- ✓ No crosstalk + +**Duration**: ~8 seconds + +--- + +### TC04.6 - Flap Pattern Comparison + +**Procedure**: +1. Get 2 interfaces (damped and undamped) +2. Clear statistics +3. Generate flap pattern A: 5 rapid flaps (0.3s interval) + 5s pause + 2 sparse flaps +4. Observe event propagation on both interfaces +5. Compare propagated events + +**Expected Results**: +- ✓ Damped interface suppresses more events (if pattern triggers threshold) +- ✓ Undamped interface propagates all events +- ✓ Counters show clear difference + +**Duration**: ~20 seconds + +--- + +### TC04.7 - Large-Scale Mixed Configuration + +**Procedure**: +1. Get 10 interfaces +2. Alternately enable/disable damping: + - Interfaces 1, 3, 5, 7, 9: damping enabled + - Interfaces 2, 4, 6, 8, 10: damping disabled +3. Verify all 10 configurations independently +4. Verify CONFIG_DB has 10 entries (or subset per topology) + +**Expected Results**: +- ✓ All 10 interfaces configured correctly +- ✓ Alternating pattern verified +- ✓ All configurations persisted +- ✓ No configuration errors +- ✓ System handles 10+ configurations + +**Duration**: ~5 seconds + +--- + +## TC05: Post-Damping Operational State Accuracy + +**Objective**: Ensure operational state reflects physical state after damping ends. + +### TC05.1 - Operational State Frozen During Suppression + +**Procedure**: +1. Configure damping on test interface with standard parameters +2. Generate 5 rapid flaps (0.5s interval) to trigger suppression +3. **While suppression is active**: + - Record operational state + - Record physical state +4. Compare states during suppression +5. Verify physical and operational states diverge + +**Expected Results**: +- ✓ Suppression is confirmed active +- ✓ Operational state ≠ Physical state (one is frozen) +- ✓ Operational state matches last propagated event +- ✓ Physical state reflects actual link condition +- ✓ Divergence confirmed during suppression + +**Duration**: ~8 seconds + +--- + +### TC05.2 - Operational State Updates After Suppression Ends + +**Procedure**: +1. Configure damping and trigger suppression (as TC05.1) +2. Wait for suppression to end: + - max_suppress_time = 30 seconds + - Wait: 30 + 10 = 40 seconds (buffer for decay) +3. Verify suppression is no longer active +4. Record operational state +5. Record physical state +6. Compare states + +**Expected Results**: +- ✓ Suppression is no longer active +- ✓ Operational state == Physical state +- ✓ State update occurred +- ✓ States are synchronized + +**Duration**: ~50 seconds + +--- + +### TC05.3 - Physical vs Operational State Divergence During Suppression + +**Procedure**: +1. Enable damping on test interface +2. Record initial states +3. Generate 10 rapid flaps (0.3s interval) to trigger suppression +4. **During suppression period**: + - Monitor physical state (from fanout perspective) + - Monitor operational state (from DUT perspective) + - Record at t=2s, t=5s, t=10s, t=20s, t=30s +5. Create divergence timeline + +**Expected Results**: +- ✓ Divergence detected during suppression +- ✓ Physical state changes with link +- ✓ Operational state remains frozen +- ✓ Clear timeline of divergence/convergence +- ✓ States match again after suppression ends + +**Duration**: ~35 seconds + +--- + +### TC05.4 - Penalty Decay and State Recovery + +**Procedure**: +1. Enable damping and trigger suppression +2. Record initial penalty: P₀ +3. Wait 5 seconds, record penalty: P₅ +4. Wait another 5 seconds (t=10), record penalty: P₁₀ +5. Calculate decay rate +6. Compare with expected decay (half-life based) + +**Expected Results**: +- ✓ P₀ > P₅ > P₁₀ (monotonic decay) +- ✓ Decay rate approximately follows half-life formula +- ✓ Penalty eventually drops below reuse threshold +- ✓ State recovery occurs when penalty < reuse_threshold + +**Duration**: ~15 seconds + +--- + +### TC05.5 - Multiple Suppression Cycles + +**Procedure**: +1. Enable damping on test interface +2. **Cycle 1**: + - Generate 5 flaps to trigger suppression + - Wait 35 seconds for suppression to end +3. **Cycle 2**: + - Generate 5 flaps again + - Wait 35 seconds +4. Verify operational state recovery both cycles + +**Expected Results**: +- ✓ Cycle 1: Suppression starts, runs, ends correctly +- ✓ Cycle 2: Second suppression cycle behaves identically +- ✓ State recovery occurs both cycles +- ✓ No hanging state between cycles +- ✓ System can handle multiple suppression cycles + +**Duration**: ~80 seconds + +--- + +## TC06: Frequent vs Infrequent Flaps + +**Objective**: Verify proportional suppression based on flap frequency. + +### TC06.1 - Frequent Flaps Longer Suppression + +**Procedure**: +1. Enable damping on test interface +2. Generate 10 flaps with 0.5s interval (frequent, rapid) +3. Record suppression start time +4. Monitor suppression status every 2 seconds +5. Record suppression end time +6. Calculate total suppression duration: T_supp + +**Expected Results**: +- ✓ Suppression is triggered (penalty reaches threshold) +- ✓ T_supp > 10 seconds (significant suppression) +- ✓ T_supp ≤ max_suppress_time (30 seconds) +- ✓ Longer suppression for frequent flaps (higher penalty accumulation) + +**Duration**: ~40 seconds + +--- + +### TC06.2 - Infrequent Flaps Shorter Suppression + +**Procedure**: +1. Enable damping on test interface +2. Generate 2 flaps with 5 second interval (sparse, infrequent) +3. Record suppression start time (if triggered) +4. Monitor suppression status +5. Record suppression end time +6. Calculate suppression duration: T_supp + +**Expected Results**: +- ✓ Suppression may be triggered (lower penalty) +- ✓ If triggered, T_supp < suppression from TC06.1 +- ✓ Shorter suppression for sparse flaps (lower penalty) +- ✓ Suppression ends sooner + +**Duration**: ~25 seconds + +--- + +### TC06.3 - Penalty Accumulation Difference + +**Procedure**: +1. Get 2 interfaces (Interface A and B) +2. Enable damping on both with same parameters +3. **Interface A**: Generate 10 flaps with 0.3s interval (frequent) +4. **Interface B**: Generate 2 flaps with 5s interval (sparse) +5. Record penalty on both interfaces immediately after flaps +6. Compare penalty values: P_A vs P_B + +**Expected Results**: +- ✓ P_A > P_B (frequent flaps accumulate more penalty) +- ✓ Significant difference in penalty values +- ✓ Demonstrates penalty accumulation difference + +**Duration**: ~12 seconds + +--- + +### TC06.4 - Decay Rate Same for Both + +**Procedure**: +1. Two interfaces with same configuration +2. Trigger suppression on both (different flap patterns) +3. Record penalties at t=0, t=5, t=10, t=15 +4. Interface A penalties: P_A0, P_A5, P_A10, P_A15 +5. Interface B penalties: P_B0, P_B5, P_B10, P_B15 +6. Calculate decay rates (ratio of penalties over time) + +**Expected Results**: +- ✓ Decay rate (P_t / P_0) is same for both interfaces +- ✓ Decay follows same mathematical formula (exponential) +- ✓ At t=decay_half_life (15s): P_t ≈ P_0 / 2 for both +- ✓ Decay rate independent of initial penalty + +**Duration**: ~20 seconds + +--- + +### TC06.5 - Recovery Time Proportional to Frequency + +**Procedure**: +1. Two interfaces with same parameters +2. Interface A: Frequent flaps (10 with 0.3s interval) +3. Interface B: Sparse flaps (2 with 5s interval) +4. Measure recovery time for each: + - From suppression start to end (when penalty < reuse_threshold) +5. Compare recovery times: T_A vs T_B + +**Expected Results**: +- ✓ T_A > T_B (frequent flaps take longer to recover) +- ✓ Recovery time proportional to peak penalty +- ✓ Higher penalty = longer recovery +- ✓ Clear correlation between frequency and recovery time + +**Duration**: ~50 seconds + +--- + +### TC06.6 - Mixed Pattern Suppression + +**Procedure**: +1. Enable damping on test interface +2. Generate mixed pattern: + - Phase 1: 5 rapid flaps (0.3s interval) - **frequent** + - Pause 5 seconds + - Phase 2: 2 sparse flaps (5s interval apart) - **sparse** +3. Monitor penalties during all phases +4. Track suppression status + +**Expected Results**: +- ✓ Phase 1 triggers suppression (high penalty) +- ✓ Phase 2 occurs while penalty is still decaying +- ✓ Penalty may spike during Phase 2 +- ✓ Suppression duration reflects combined pattern +- ✓ System handles pattern transitions correctly + +**Duration**: ~30 seconds + +--- + +### TC06.7 - Threshold Crossing Different Timing + +**Procedure**: +1. Two interfaces with different configurations: + - Interface A: suppress_threshold = 1600 (standard) + - Interface B: suppress_threshold = 3200 (conservative, higher threshold) +2. Both interfaces: Generate 10 flaps with 0.5s interval +3. Record time to reach suppress_threshold for each +4. Compare threshold crossing times + +**Expected Results**: +- ✓ Interface A crosses threshold sooner (lower threshold) +- ✓ Interface B takes longer (higher threshold) +- ✓ Clear timing difference in suppression start +- ✓ Demonstrates threshold impact on suppression timing + +**Duration**: ~12 seconds + +--- + +## TC07: Stats Verification + +**Objective**: Validate accuracy of link damping counters. + +### TC07.1 - Pre-Damping Link Transitions Counter + +**Procedure**: +1. Enable damping on test interface +2. Clear all statistics +3. Generate 5 link flaps +4. Retrieve counter: `pre_damping_link_transitions` +5. Verify counter value + +**Expected Results**: +- ✓ Counter incremented for each transition +- ✓ pre_damping_link_transitions ≥ 5 (minimum 5 transitions from 5 flaps) +- ✓ Counter is accurate + +**Duration**: ~5 seconds + +--- + +### TC07.2 - Post-Damping Propagated Transitions Counter + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 10 flaps with 0.5s interval +4. Retrieve counter: `post_damping_propagated_transitions` +5. Verify counter value + +**Expected Results**: +- ✓ Counter incremented only for non-suppressed events +- ✓ post_damping_propagated_transitions ≤ pre_damping_link_transitions +- ✓ Counter reflects only advertised events + +**Duration**: ~8 seconds + +--- + +### TC07.3 - Pre-Damping UP Events Counter + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 5 flaps (each flap = 1 DOWN + 1 UP event) +4. Retrieve counter: `pre_damping_up_events` +5. Verify counter value + +**Expected Results**: +- ✓ pre_damping_up_events ≥ 5 (one UP per flap) +- ✓ Counter reflects actual UP transitions +- ✓ Accurate UP event count + +**Duration**: ~5 seconds + +--- + +### TC07.4 - Pre-Damping DOWN Events Counter + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 5 flaps (each flap = 1 DOWN event initially) +4. Retrieve counter: `pre_damping_down_events` +5. Verify counter value + +**Expected Results**: +- ✓ pre_damping_down_events ≥ 5 (one DOWN per flap) +- ✓ Counter reflects actual DOWN transitions +- ✓ Accurate DOWN event count + +**Duration**: ~5 seconds + +--- + +### TC07.5 - Post-Damping UP Advertised Counter + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 10 flaps with 0.5s interval +4. Retrieve counter: `post_damping_up_advertised` +5. Verify counter value reflects only advertised UPs + +**Expected Results**: +- ✓ post_damping_up_advertised ≤ pre_damping_up_events +- ✓ Only non-suppressed UP events counted +- ✓ Counter is accurate + +**Duration**: ~8 seconds + +--- + +### TC07.6 - Post-Damping DOWN Advertised Counter + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 10 flaps with 0.5s interval +4. Retrieve counter: `post_damping_down_advertised` +5. Verify counter value reflects only advertised DOWNs + +**Expected Results**: +- ✓ post_damping_down_advertised ≤ pre_damping_down_events +- ✓ Only non-suppressed DOWN events counted +- ✓ Counter is accurate + +**Duration**: ~8 seconds + +--- + +### TC07.7 - Counter Consistency Across Cycles + +**Procedure**: +1. Enable damping +2. Clear statistics +3. **Cycle 1**: Generate 5 flaps, record counters: C1 +4. **Cycle 2**: Generate 3 more flaps, record counters: C2 +5. Compare: C2 should be C1 + new increments + +**Expected Results**: +- ✓ Counters increment monotonically +- ✓ C2_transitions = C1_transitions + new_transitions +- ✓ No counter resets between cycles +- ✓ Counters remain consistent + +**Duration**: ~10 seconds + +--- + +### TC07.8 - Counter Increments Proportional to Events + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 10 flaps with 0.5s interval +4. Retrieve counter: `pre_damping_link_transitions` +5. Verify counter proportional to number of flaps + +**Expected Results**: +- ✓ pre_damping_link_transitions ≥ 10 (or ≥ expected count) +- ✓ Counter increments with each event +- ✓ Linear relationship: more flaps = higher counter +- ✓ Counter accuracy maintained + +**Duration**: ~8 seconds + +--- + +### TC07.9 - Suppressed Events Not in Post-Damping + +**Procedure**: +1. Enable damping +2. Clear statistics +3. Generate 20 flaps with 0.3s interval (sufficient to trigger suppression) +4. Retrieve both counters: + - pre_damping_link_transitions + - post_damping_propagated_transitions +5. Calculate difference + +**Expected Results**: +- ✓ pre_damping > post_damping (some events suppressed) +- ✓ Difference = number of suppressed events +- ✓ Suppressed events NOT counted in post-damping +- ✓ Clear evidence of suppression + +**Duration**: ~8 seconds + +--- + +### TC07.10 - Counter Reset and Recovery + +**Procedure**: +1. Enable damping +2. Generate 5 flaps +3. Retrieve counters: initial_count +4. Clear statistics (reset counters to 0) +5. Retrieve counters: should be ~0 +6. Generate 3 more flaps +7. Retrieve counters: should equal 3 (or proportional count) + +**Expected Results**: +- ✓ Counters reset successfully +- ✓ After reset: counters ≈ 0 +- ✓ New events counted from 0 +- ✓ Counter recovery works correctly +- ✓ No stale data from previous count + +**Duration**: ~10 seconds + +--- + +## TC09: Timeline Validation + +**Objective**: Validate link damping algorithm using deterministic event sequence per HLD. + +**Note**: This comprehensive test validates the core damping algorithm against the HLD timeline specification. + +### TC09.1 - Timeline Event Sequence Execution + +**Configuration**: +``` +suppress_threshold: 1600 +reuse_threshold: 1200 +decay_half_life: 15 seconds +max_suppress_time: 30 seconds +flap_penalty: 500 +``` + +**Procedure**: +1. Enable damping with above configuration +2. Clear statistics +3. Execute deterministic timeline of events (starting at t=0): + +| Time | Event | Expected Propagated | +|------|-------|------------------| +| 3s | DOWN | Yes (pre-threshold) | +| 7s | UP | Yes (pre-threshold) | +| 10s | DOWN | Yes (accumulated penalty=500) | +| 14s | UP | No (penalty=1000, < threshold) | +| 17s | DOWN | No (suppression active) | +| 20s | UP | No (suppression active) | +| 31s | - | Yes (penalty < reuse, suppression ends) | +| 40s | DOWN | Yes (new cycle) | +| 44s | UP | No (suppression active) | +| 46s | DOWN | No (suppression active) | +| 61s | - | No (suppression ending) | +| 70s | UP | Yes (suppression ended) | +| 100s | DOWN | Yes (no suppression) | +| 102s | UP | Yes (no suppression) | +| 105s | DOWN | Yes (suppression starts) | +| 124s | UP | No (suppression active) | +| 152s | - | Yes (suppression ends) | + +**Expected Results**: +- ✓ All events execute at correct times (within 1 second tolerance) +- ✓ Suppression starts at correct thresholds +- ✓ Events are suppressed/propagated per timeline +- ✓ Suppression ends at correct times +- ✓ Multiple suppression cycles work correctly +- ✓ Algorithm matches HLD specification + +**Duration**: ~160 seconds + + +--- + +## TC10: Persistence and Restart Resilience + +**Objective**: Verify damping configuration and functionality persists across reboots and docker restarts. + +### TC10.1 - Damping Config Persists After Reboot + +**Procedure**: +1. Select test interface +2. Configure damping with standard parameters +3. Verify configuration in CONFIG_DB +4. Reboot DUT: `sudo reboot` +5. Wait for DUT to come up (~3-5 minutes) +6. Query CONFIG_DB again +7. Compare pre/post reboot configurations + +**Expected Results**: +- ✓ Configuration exists before reboot +- ✓ Reboot completes successfully +- ✓ Configuration persists in CONFIG_DB after reboot +- ✓ All parameters match pre-reboot values +- ✓ No configuration loss + +**Duration**: ~5-10 minutes + +--- + +### TC10.2 - Damping Functionality After Reboot + +**Procedure**: +1. Configure damping (as TC10.1) +2. Reboot DUT +3. Wait for DUT to be fully up +4. Clear statistics +5. Generate 10 flaps with 0.5s interval +6. Retrieve counters: + - pre_damping_link_transitions + - post_damping_propagated_transitions +7. Verify suppression is working (post < pre) + +**Expected Results**: +- ✓ Configuration persists +- ✓ Damping is fully operational after reboot +- ✓ Suppression works correctly +- ✓ Counters are accurate +- ✓ No functional degradation + +**Duration**: ~5-10 minutes + 8 seconds + +--- + +### TC10.3 - Counters Preserved After Reboot + +**Procedure**: +1. Configure damping and trigger events +2. Generate 5 flaps +3. Retrieve counters: C_before +4. Reboot DUT +5. Wait for DUT to come up +6. Retrieve counters: C_after +7. Compare counter values + +**Expected Results**: +- ✓ Counters before reboot: C_before > 0 +- ✓ Counters after reboot: C_after (may or may not persist - depends on implementation) +- ✓ If persisted: C_after ≥ C_before +- ✓ No counter corruption + +**Duration**: ~5-10 minutes + +--- + +### TC10.4 - Multiple Reboot Cycles + +**Procedure**: +1. Configure damping on test interface +2. **Reboot Cycle 1**: + - Verify configuration exists + - Reboot DUT + - Verify configuration persists + - Verify functionality works +3. **Reboot Cycle 2**: + - Repeat cycle 1 steps + +**Expected Results**: +- ✓ Cycle 1: Configuration survives reboot +- ✓ Cycle 2: Configuration survives second reboot +- ✓ Multiple reboots handled correctly +- ✓ No progressive degradation +- ✓ System is stable + +**Duration**: ~10-20 minutes + +--- + +### TC10.5 - Concurrent Damping Multiple Ports After Reboot + +**Procedure**: +1. Configure damping on 5 interfaces with different parameters: + - Interface 1-3: standard config + - Interface 4-5: conservative config (higher thresholds) +2. Reboot DUT +3. Wait for recovery +4. Verify all 5 configurations persisted +5. Generate flaps on each interface +6. Verify independent suppression on each + +**Expected Results**: +- ✓ All 5 configurations survive reboot +- ✓ Configurations are independent +- ✓ Suppression works correctly on each +- ✓ No cross-interference +- ✓ System handles concurrent damping post-reboot + +**Duration**: ~5-10 minutes + +--- + +### TC10.6 - Reboot During Suppression + +**Procedure**: +1. Configure damping on test interface +2. Generate flaps to trigger suppression +3. Verify suppression is active +4. Immediately reboot DUT (while suppressed) +5. Wait for DUT to come up +6. Verify configuration persisted +7. Verify system stability + +**Expected Results**: +- ✓ Reboot is graceful (no hang) +- ✓ Configuration persists +- ✓ DUT comes up fully +- ✓ Suppression state is reset (clean state) +- ✓ Functionality works after recovery +- ✓ No state corruption + +**Duration**: ~5-10 minutes + +--- + +### TC10.7 - BGP Docker Restart + +**Procedure**: +1. Configure damping on test interface +2. Verify configuration in CONFIG_DB +3. Restart BGP container: `docker restart bgp` +4. Wait for BGP to fully restart (~10-30 seconds) +5. Verify configuration persists +6. Generate flaps and verify suppression works + +**Expected Results**: +- ✓ BGP restart completes successfully +- ✓ Configuration persists in CONFIG_DB +- ✓ Damping functionality unaffected +- ✓ No ASIC state issues +- ✓ System continues operating + +**Duration**: ~1-2 minutes + +--- + +### TC10.8 - SWSS Docker Restart + +**Procedure**: +1. Configure damping on test interface +2. Verify configuration in CONFIG_DB and APP_DB +3. Restart SWSS container: `docker restart swss` +4. Wait for SWSS to fully restart and reconcile (~30-60 seconds) +5. Verify configuration persists in both databases +6. Generate flaps and verify suppression works +7. Check for any ASIC inconsistencies + +**Expected Results**: +- ✓ SWSS restart completes successfully +- ✓ Configuration persists in CONFIG_DB +- ✓ Configuration reconciled in APP_DB +- ✓ Damping fully operational post-restart +- ✓ No ASIC inconsistency +- ✓ All counters reset (fresh state) + +**Duration**: ~2-3 minutes + +--- + +### TC10.9 - Syncd Docker Restart + +**Procedure**: +1. Configure damping on test interface +2. Verify configuration and ASIC state +3. Restart Syncd container: `docker restart syncd` +4. Wait for Syncd to fully reconnect to ASIC (~20-60 seconds) +5. Verify configuration persists +6. Verify ASIC state remains consistent +7. Generate flaps and verify suppression works +8. Check ASIC counters/state + +**Expected Results**: +- ✓ Syncd restart completes successfully +- ✓ ASIC reconnection successful +- ✓ Configuration persists +- ✓ ASIC state consistent (no mismatch) +- ✓ Damping functionality intact +- ✓ No counter corruption +- ✓ Link states correct after restart + +**Duration**: ~2-3 minutes + +--- + +## Test Execution Summary + +### Quick Reference: Test Count and Duration + +| Test Class | Test Count | Total Duration | +|-----------|-----------|-----------------| +| TC01 - Basics | 3 | ~15 seconds | +| TC02 - Configuration | 6 | ~12 seconds | +| TC03 - Unsupported | 6 | ~25 seconds | +| TC04 - Mixed Config | 6 | ~35 seconds | +| TC05 - Operational State | 5 | ~150 seconds | +| TC06 - Frequency Effects | 7 | ~150 seconds | +| TC07 - Counters | 10 | ~80 seconds | +| TC09 - Timeline | 1 | ~160 seconds | +| TC10 - Persistence | 9 | ~45-60 minutes | +| **TOTAL** | **53** | **~46-62 minutes** | + +**Note**: Most of TC10 tests require device reboot (5-10 min each), which is why the total time is significant. + +--- + +## Running Tests Efficiently + +### For Quick Validation (excluding reboots) +```bash +# Run all tests except TC10 (persistence) +pytest tests/link_event_damping/test_link_event_damping.py -v -k "not tc10" + +# Estimated time: ~10-15 minutes +``` + +### For Overnight Testing +```bash +# Run all tests including persistence/reboots +pytest tests/link_event_damping/test_link_event_damping.py -v + +# Estimated time: ~60-80 minutes (depending on DUT boot time) +``` + +### For Specific Feature Testing +```bash +# Test suppression logic +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingOperationalState -v + +# Test configuration validation +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingConfiguration -v + +# Test counters +pytest tests/link_event_damping/test_link_event_damping.py::TestLinkEventDampingCounters -v +``` + +### Parallel Execution (if supported by test infrastructure) +```bash +# Run on multiple DUTs +pytest tests/link_event_damping/test_link_event_damping.py -v -n 4 + +# Note: Adjust based on available resources +``` + +--- + +## Expected Pass Criteria + +### Comprehensive Success +- ✓ All 53 test cases pass +- ✓ No critical errors in logs +- ✓ Configuration persists correctly +- ✓ Counters are accurate +- ✓ Suppression algorithm matches HLD +- ✓ System stable before/after all tests + +### Partial Success (acceptable for feature development) +- ✓ TC01-TC07 pass (basic functionality) +- ✓ TC09 passes (algorithm validation) +- ✗ TC10 may have issues if restart feature not complete + +### Known Limitations +- Some TC10 tests may timeout if DUT boot time > 5 minutes +- Fanout-based flaps require proper topology; fallback to admin commands works +- Counter persistence across reboot depends on implementation + +--- + +## Troubleshooting + +### Common Issues and Solutions + +#### 1. Test Timeout +**Symptom**: Test hangs waiting for suppression to end +**Cause**: Decay half-life calculation issue +**Solution**: +```bash +# Check penalty decay: +redis-cli -n 1 HGET 'LINK_DAMPING_STATUS:Ethernet0' current_penalty +# Wait and check again +``` + +#### 2. Configuration Not Applied +**Symptom**: CONFIG_DB shows no entries +**Cause**: Configuration command syntax error +**Solution**: +```bash +# Verify supported syntax: +config interface link-damping --help +# Check SWSS logs: +docker exec -it swss tail -100 /var/log/swss.log +``` + +#### 3. Counters Not Incrementing +**Symptom**: All counters remain 0 +**Cause**: Link events not being generated properly +**Solution**: +```bash +# Verify link flap is working: +show interfaces status +# Check if events are in syslog: +grep -i "link\|flap" /var/log/syslog | tail -20 +``` + +#### 4. Fanout Connection Issues +**Symptom**: Flap generation fails +**Cause**: Fanout switch not accessible +**Solution**: +```bash +# Test will fall back to DUT admin commands +# Verify manual flap: +config interface shutdown Ethernet0 +config interface startup Ethernet0 +``` + +--- + +## Validation Checklist + +Before declaring Link Event Damping feature complete: + +- [ ] All 53 tests pass +- [ ] Configuration persists correctly +- [ ] Suppression algorithm validates against HLD timeline +- [ ] Counters are accurate and consistent +- [ ] Reboot resilience verified (TC10.1-TC10.6) +- [ ] Docker restart resilience verified (TC10.7-TC10.9) +- [ ] Multi-port concurrent damping works +- [ ] No memory leaks or hangs +- [ ] Performance acceptable (suppression/recovery times match HLD) +- [ ] No unexpected errors in system logs + +--- + +## References + +- SONiC Link Event Damping HLD: `/home/hp_test/tsiva/Link_event_damping.md` +- Test Suite: `tests/link_event_damping/test_link_event_damping.py` +- Utilities: `tests/link_event_damping/link_event_damping_utils.py` +- Configuration: `tests/link_event_damping/conftest.py` + +--- + +## Contact & Support + +For test execution issues or clarifications: +1. Check test logs: `pytest ... -v -s` (verbose + show output) +2. Check system logs: `/var/log/syslog`, `/var/log/swss.log` +3. Check Redis databases: `redis-cli -n KEYS '*DAMPING*'` +4. Review HLD specification for algorithm details diff --git a/tests/link_dampening/conftest.py b/tests/link_dampening/conftest.py new file mode 100644 index 00000000000..a54d3bfcf5a --- /dev/null +++ b/tests/link_dampening/conftest.py @@ -0,0 +1,93 @@ +""" +Pytest configuration and fixtures for Layer1 (Physical) tests +""" + +import logging +import pytest + +#from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.assertions import pytest_assert as pt_assert +from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports + +from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def link_dampening_test_interface(duthosts, enum_rand_one_per_hwsku_frontend_hostname, conn_graph_facts, tbinfo): + """ + Get a test interface that is eligible for link_dampening testing. + + Returns the first available front-facing interface on the DUT. + """ + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + # Get all front-facing interfaces + front_interfaces = get_dut_fronface_ports(dut, tbinfo) + + #pytest_assert(front_interfaces, "No front-facing interfaces found on DUT") + + # Return the first interface + return front_interfaces[0] + + +@pytest.fixture(scope="module") +def link_dampening_test_interfaces(duthosts, enum_rand_one_per_hwsku_frontend_hostname, conn_graph_facts, tbinfo): + """ + Get multiple test interfaces for link_dampening testing. + + Returns up to 5 available front-facing interfaces. + """ + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + # Get all front-facing interfaces + front_interfaces = get_dut_fronface_ports(dut, tbinfo) + + #pytest_assert(len(front_interfaces) >= 2, "Need at least 2 front-facing interfaces") + + # Return up to 5 interfaces + return front_interfaces[:5] + + +@pytest.fixture(scope="function") +def cleanup_link_damping(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """ + Cleanup fixture that ensures link damping stats are cleared after each test. + """ + yield + + # Cleanup code runs after test + try: + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Clear link damping stats + dut.shell("redis-cli -n 0 --scan --match 'LINK_DAMPING_STATS:*' | xargs redis-cli -n 0 DEL", + module_ignore_errors=True) + logger.info("Cleaned up link damping statistics") + except Exception as e: + logger.warning(f"Could not cleanup link damping stats: {e}") + + +@pytest.fixture(scope="module") +def get_test_interface_from_graph(conn_graph_facts, tbinfo): + """ + Get test interface information from connection graph. + + Useful for getting fanout interface information. + """ + def _get_interface(dut_hostname, dut_interface): + """Get fanout interface connected to DUT interface.""" + try: + # Find the connection from graph + for link_name, link_info in conn_graph_facts.items(): + for intf_a, intf_b in link_info.items(): + if (intf_a[0] == dut_hostname and intf_a[1] == dut_interface): + return intf_b[0], intf_b[1] # (fanout_hostname, fanout_interface) + elif (intf_b[0] == dut_hostname and intf_b[1] == dut_interface): + return intf_a[0], intf_a[1] # (fanout_hostname, fanout_interface) + except Exception as e: + logger.warning(f"Could not get fanout interface for {dut_interface}: {e}") + + return None, None + + return _get_interface diff --git a/tests/link_dampening/link_event_damping_utils.py b/tests/link_dampening/link_event_damping_utils.py new file mode 100644 index 00000000000..4042b81cd89 --- /dev/null +++ b/tests/link_dampening/link_event_damping_utils.py @@ -0,0 +1,712 @@ +""" +Utility functions for Link Event Damping tests + +This module provides helper functions for: +- Configuring link damping parameters +- Generating link flaps via Fanout switch +- Collecting and verifying link damping statistics +- Managing Redis database state +""" + +import logging +import time +import json +import re +from datetime import datetime +from natsort import natsorted + +logger = logging.getLogger(__name__) + + +def get_dut_fronface_ports(duthost, tbinfo): + """ + Get all front-facing (non-backend) ports from the DUT. + + Args: + duthost: The AnsibleHost object of DUT. + tbinfo: Testbed information dictionary. + + Returns: + list: List of front-facing interface names (e.g., ['Ethernet0', 'Ethernet4', ...]) + """ + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + front_ports = [] + + for port in mg_facts.get('minigraph_ports', {}).keys(): + # Exclude backend ports (contain 'Ethernet-BP' or have non-Ext role) + if not duthost.is_backend_port(port, mg_facts): + front_ports.append(port) + + return natsorted(front_ports) + + +def configure_link_damping(dut, interface, suppress_threshold=None, reuse_threshold=None, + decay_half_life=None, max_suppress_time=None, flap_penalty=None, + algorithm="aied", disabled=False): + """ + Configure link damping parameters on a specific interface using AIED algorithm. + + Args: + dut: DUT host object + interface: Interface name (e.g., "Ethernet0") + suppress_threshold: Penalty threshold to enter damped state + reuse_threshold: Penalty threshold to exit damped state + decay_half_life: Time in milliseconds for penalty to decay by half + max_suppress_time: Maximum time in milliseconds an interface can be suppressed + flap_penalty: Penalty added per link down event + algorithm: Damping algorithm to use (default: "aied") + disabled: Set to True to disable damping on the interface + + Returns: + bool: True if configuration successful + """ + try: + if disabled: + # Disable damping on the interface by removing algorithm + # TODO: Verify the correct command to disable damping + cmd = f"config interface damping algo {interface} disabled" + result = dut.shell(cmd, module_ignore_errors=True) + logger.info(f"Link damping disabled on {interface}") + return True + + # Step 1: Configure the damping algorithm + algo_cmd = f"config interface damping algo {interface} {algorithm}" + result = dut.shell(algo_cmd, module_ignore_errors=True) + + if result["rc"] != 0: + logger.warning(f"Failed to configure damping algorithm on {interface}: {result.get('stderr', '')}") + return False + + logger.info(f"Damping algorithm '{algorithm}' configured on {interface}") + + # Step 2: Configure damping parameters if any are provided + if any([suppress_threshold, reuse_threshold, decay_half_life, max_suppress_time, flap_penalty]): + # Build parameter configuration command + param_cmd = f"config interface damping {algorithm}-param {interface}" + + if suppress_threshold is not None: + param_cmd += f" --suppress-threshold {suppress_threshold}" + if reuse_threshold is not None: + param_cmd += f" --reuse-threshold {reuse_threshold}" + if decay_half_life is not None: + param_cmd += f" --decay-half-life {decay_half_life}" + if max_suppress_time is not None: + param_cmd += f" --max-suppress-time {max_suppress_time}" + if flap_penalty is not None: + param_cmd += f" --flap-penalty {flap_penalty}" + + # Execute parameter configuration + result = dut.shell(param_cmd, module_ignore_errors=True) + + if result["rc"] == 0: + logger.info(f"Link damping parameters configured on {interface}: " + f"threshold={suppress_threshold}, reuse={reuse_threshold}, " + f"decay={decay_half_life}ms, max_suppress={max_suppress_time}ms, penalty={flap_penalty}") + return True + else: + logger.warning(f"Failed to configure damping parameters on {interface}: {result.get('stderr', '')}") + return False + + return True + + except Exception as e: + logger.error(f"Error configuring link damping on {interface}: {e}") + return False + + +def verify_configuration(dut, interface, config_params): + """ + Verify that link damping configuration is applied correctly. + + Args: + dut: DUT host object + interface: Interface name + config_params: Dictionary of expected configuration parameters + + Returns: + bool: True if configuration matches expected values + """ + try: + # Query CONFIG_DB via redis-cli + cmd = f"redis-cli -n 4 HGETALL 'PORT|{interface}'" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] != 0: + logger.warning(f"Could not query CONFIG_DB for {interface}") + return False + + # Parse output + output = result["stdout"] + if not output or "no such key" in output.lower(): + logger.warning(f"No CONFIG_DB entry found for {interface}") + return False + + logger.debug(f"CONFIG_DB entry for {interface}: {output}") + return True + + except Exception as e: + logger.error(f"Error verifying configuration for {interface}: {e}") + return False + + +def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, num_flaps=1, interval=1): + """ + Generate link flaps by shutting down/bringing up fanout switch interface. + + This is the proper way to generate link flaps from the remote end (Fanout). + Link up/down events are generated at the ASIC layer via the fanout switch. + + Args: + dut: DUT host object + dut_interface: Interface name on DUT (for logging) + fanout: Fanout switch host object (optional) + fanout_interface: Interface on fanout switch to toggle (optional) + num_flaps: Number of flaps to generate + interval: Time in seconds between flaps + + Returns: + bool: True if flaps were generated successfully + """ + try: + if not fanout or not fanout_interface: + # Fallback: use DUT interface admin down/up + logger.warning("Using DUT interface for flap generation (fanout not available)") + logger.info(f"Generating {num_flaps} link flaps on {dut_interface} with {interval}s interval") + + for flap_num in range(num_flaps): + # Admin down + logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Admin down on {dut_interface}") + dut.shell(f"config interface shutdown {dut_interface}", module_ignore_errors=True) + time.sleep(interval / 2) + + # Admin up + logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Admin up on {dut_interface}") + dut.shell(f"config interface startup {dut_interface}", module_ignore_errors=True) + time.sleep(interval / 2) + + return True + + else: + # Use fanout switch interface (preferred method) + logger.info(f"Generating {num_flaps} link flaps on {dut_interface} " + f"(fanout {fanout_interface}) with {interval}s interval") + + for flap_num in range(num_flaps): + # Shut down fanout interface (causes link DOWN on DUT) + logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Shutting down {fanout_interface}") + try: + fanout.shutdown([fanout_interface]) + except: + # Fallback to shell command + fanout.shell(f"configure terminal", module_ignore_errors=True) + fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) + fanout.shell(f"shutdown", module_ignore_errors=True) + fanout.shell(f"end", module_ignore_errors=True) + + time.sleep(interval / 2) + + # Bring up fanout interface (causes link UP on DUT) + logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Bringing up {fanout_interface}") + try: + fanout.no_shutdown([fanout_interface]) + except: + # Fallback to shell command + fanout.shell(f"configure terminal", module_ignore_errors=True) + fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) + fanout.shell(f"no shutdown", module_ignore_errors=True) + fanout.shell(f"end", module_ignore_errors=True) + + time.sleep(interval / 2) + + logger.info(f"Successfully generated {num_flaps} link flaps") + return True + + except Exception as e: + logger.error(f"Error generating link flaps: {e}") + return False + + +def get_interface_operational_state(dut, interface): + """ + Get the operational state of an interface. + + Args: + dut: DUT host object + interface: Interface name + + Returns: + str: "up" or "down" + """ + try: + output = dut.show_and_parse(f"show interfaces status {interface}") + if output: + oper_state = output[0].get("oper", "unknown").lower() + return oper_state + return "unknown" + except Exception as e: + logger.error(f"Error getting operational state for {interface}: {e}") + return "unknown" + + +def get_interface_physical_state(dut, interface): + """ + Get the physical state of an interface. + + Args: + dut: DUT host object + interface: Interface name + + Returns: + str: "up" or "down" + """ + try: + # Physical state is typically the same as admin state in most cases + # unless there's a link issue + output = dut.show_and_parse(f"show interfaces status {interface}") + if output: + # In SONiC, physical state is often the real link state + phys_state = output[0].get("oper", "unknown").lower() + return phys_state + return "unknown" + except Exception as e: + logger.error(f"Error getting physical state for {interface}: {e}") + return "unknown" + + +def get_link_damping_stats(dut, interface): + """ + Retrieve link damping statistics for an interface. + + Stats include: + - pre_damping_link_transitions: Total link transitions before damping + - pre_damping_up_events: Total UP events before damping + - pre_damping_down_events: Total DOWN events before damping + - post_damping_propagated_transitions: Events propagated after damping + - post_damping_up_advertised: UP events advertised after damping + - post_damping_down_advertised: DOWN events advertised after damping + + Args: + dut: DUT host object + interface: Interface name + + Returns: + dict: Dictionary of statistics + """ + try: + # Query APPL_DB for damping statistics + # In SONiC, link damping stats are stored in APP_DB + get_oid_cmd = f"redis-cli -n 2 HGET 'COUNTERS_PORT_NAME_MAP' '{interface}'" + + oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) + + # Extract and clean the OID from the shell output + # (Assuming dut.shell returns an object where .stdout or .strip() gets the raw string) + oid = oid_result['stdout'].strip() + + # Step 2: Query APP_DB using the retrieved OID for damping statistics + # In SONiC, link damping stats are stored in APP_DB using the port's OID + cmd = f'redis-cli -n 6 HGETALL "LINK_EVENT_DAMPING_STATS|{oid}"' + + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0 and result["stdout"]: + # Parse the HGETALL output (alternating key-value pairs) + lines = result["stdout"].strip().split('\n') + stats = {} + for i in range(0, len(lines), 2): + if i + 1 < len(lines): + key = lines[i].strip().strip('"') + value = lines[i + 1].strip().strip('"') + stats[key] = value + logger.debug(f"Link damping stats for {interface}: {stats}") + return stats + else: + logger.warning(f"No statistics found for {interface}") + return {} + + except Exception as e: + logger.error(f"Error getting link damping stats for {interface}: {e}") + return {} + + +def clear_link_damping_stats(dut): + """ + Clear link damping statistics for all interfaces or specific interface. + + Args: + dut: DUT host object + + Returns: + bool: True if stats were cleared successfully + """ + try: + # Clear all link damping stats from APP_DB + cmd = "redis-cli -n 6 --scan --pattern 'LINK_EVENT_DAMPING_STATS*' | xargs redis-cli -n 6 DEL" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0: + logger.info("Link damping statistics cleared") + return True + else: + logger.warning(f"Failed to clear statistics: {result['stderr']}") + return False + + except Exception as e: + logger.error(f"Error clearing link damping stats: {e}") + return False + + +def get_redis_db_entries(dut, db_name, key_pattern): + """ + Get entries from a Redis database matching a pattern. + + Args: + dut: DUT host object + db_name: Database name (e.g., "CONFIG_DB", "APP_DB") + key_pattern: Key pattern to search (e.g., "*LINK_DAMPING*") + + Returns: + dict: Dictionary of matching entries + """ + try: + # Map database names to their indices + db_map = { + "CONFIG_DB": 6, + "APP_DB": 0, + "STATE_DB": 1, + "ASIC_DB": 2, + "COUNTER_DB": 3 + } + + db_index = db_map.get(db_name, 4) + + # Use redis-cli to scan for matching keys + cmd = f"redis-cli -n {db_index} --scan --pattern '{key_pattern}'" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0: + keys = result["stdout"].strip().split('\n') + entries = {} + for key in keys: + if key.strip(): + # Get the entry + get_cmd = f"redis-cli -n {db_index} HGETALL '{key.strip()}'" + get_result = dut.shell(get_cmd, module_ignore_errors=True) + if get_result["rc"] == 0: + entries[key.strip()] = get_result["stdout"] + + logger.debug(f"Found {len(entries)} entries in {db_name}") + return entries + else: + logger.warning(f"Failed to query {db_name}") + return {} + + except Exception as e: + logger.error(f"Error getting Redis entries: {e}") + return {} + + +def validate_redis_persistence(dut, interface, config_params): + """ + Validate that configuration is persisted in Redis databases. + + Args: + dut: DUT host object + interface: Interface name + config_params: Expected configuration parameters + + Returns: + bool: True if configuration is persisted correctly + """ + try: + # Check CONFIG_DB + get_oid_cmd = f"redis-cli -n 2 HGET 'COUNTERS_PORT_NAME_MAP' '{interface}'" + + oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) + oid = oid_result['stdout'].strip() + config_entries = get_redis_db_entries(dut, "CONFIG_DB", f"*LINK_EVENT_DAMPING*{oid}*") + + if not config_entries: + logger.warning(f"No CONFIG_DB entries found for {interface}") + return False + + logger.info(f"Configuration persisted in Redis for {interface}") + return True + + except Exception as e: + logger.error(f"Error validating Redis persistence: {e}") + return False + + +def get_dampening_penalties(dut, interface): + """ + Get current dampening penalty value for an interface. + + Args: + dut: DUT host object + interface: Interface name + + Returns: + int: Current penalty value (0 if not in damping state) + """ + try: + # Query STATE_DB for current penalty + get_oid_cmd = f"redis-cli -n 2 HGET 'COUNTERS_PORT_NAME_MAP' '{interface}'" + + oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) + oid = oid_result['stdout'].strip() + + cmd = f"redis-cli -n 6 HGET 'LINK_EVENT_DAMPING_STATS|{oid}' 'current_penalty'" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0 and result["stdout"]: + penalty = int(result["stdout"].strip().strip('"')) + logger.debug(f"Current penalty for {interface}: {penalty}") + return penalty + else: + logger.debug(f"No penalty info for {interface} (likely not in damping state)") + return 0 + + except Exception as e: + logger.warning(f"Error getting penalty for {interface}: {e}") + return 0 + + +def check_suppression_active(dut, interface): + """ + Check if suppression is currently active on an interface. + + Args: + dut: DUT host object + interface: Interface name + + Returns: + bool: True if suppression is active + """ + try: + # Query STATE_DB for suppression status + get_oid_cmd = f"redis-cli -n 2 HGET 'COUNTERS_PORT_NAME_MAP' '{interface}'" + + oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) + oid = oid_result['stdout'].strip() + + cmd = f"redis-cli -n 6 HGET 'LINK_EVENT_DAMPING_STATS|{oid}' 'is_damping_active'" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0 and result["stdout"]: + status = result["stdout"].strip().strip('"').lower() + is_active = status in ["true", "1", "yes"] + logger.debug(f"Suppression active on {interface}: {is_active}") + return is_active + else: + # Also check if penalty is above suppress threshold + penalty = get_dampening_penalties(dut, interface) + return penalty > 0 + + except Exception as e: + logger.warning(f"Error checking suppression status for {interface}: {e}") + return False + + +def verify_counter_values(dut, interface, expected_counters): + """ + Verify that counter values match expected values. + + Args: + dut: DUT host object + interface: Interface name + expected_counters: Dictionary of expected counter values + + Returns: + bool: True if all counters match expected values + """ + try: + stats = get_link_damping_stats(dut, interface) + + for counter_name, expected_value in expected_counters.items(): + actual_value = int(stats.get(counter_name, 0)) + if actual_value != expected_value: + logger.warning(f"{counter_name}: expected {expected_value}, got {actual_value}") + return False + + logger.debug(f"debug:All counters verified for {interface}") + logger.warning(f"warning:All counters verified for {interface}") + return True + + except Exception as e: + logger.error(f"Error verifying counter values: {e}") + return False + + +def calculate_expected_suppression_time(suppress_threshold, reuse_threshold, decay_half_life, penalty): + """ + Calculate expected time for suppression to end based on decay. + + Args: + suppress_threshold: Threshold to enter suppression + reuse_threshold: Threshold to exit suppression + decay_half_life: Half-life for penalty decay (in milliseconds) + penalty: Current penalty value + + Returns: + float: Expected time in seconds for suppression to end + """ + try: + if penalty <= reuse_threshold: + return 0 + + # Exponential decay: penalty(t) = penalty_0 * (0.5)^(t / half_life) + # Solve for t when penalty(t) = reuse_threshold + # t = half_life * log2(penalty_0 / reuse_threshold) + + import math + if penalty > 0 and reuse_threshold > 0: + ratio = penalty / reuse_threshold + # Convert decay_half_life from milliseconds to seconds for calculation + decay_half_life_sec = decay_half_life / 1000.0 + time_to_reuse = decay_half_life_sec * math.log2(ratio) + logger.info(f"Expected suppression time: {time_to_reuse:.2f}s " + f"(penalty {penalty} -> {reuse_threshold}, half_life={decay_half_life}ms)") + return time_to_reuse + else: + return 0 + + except Exception as e: + logger.error(f"Error calculating suppression time: {e}") + return 0 + + +def inject_traffic_and_verify(dut, ptf, interface, traffic_config, verify_callback): + """ + Inject traffic and verify behavior using spytest/scapy APIs. + + Args: + dut: DUT host object + ptf: PTF host object (for traffic generation) + interface: Interface to test + traffic_config: Traffic configuration dictionary + verify_callback: Callback function to verify results + + Returns: + bool: True if traffic was injected and verified successfully + """ + try: + logger.info(f"Injecting traffic on {interface}") + # This would use spytest scapy APIs to generate traffic + # Implementation depends on spytest framework integration + logger.info("Traffic injection and verification completed") + return True + + except Exception as e: + logger.error(f"Error injecting traffic: {e}") + return False + + +def restart_docker_container(dut, container_name): + """ + Restart a Docker container on the DUT. + + Args: + dut: DUT host object + container_name: Name of the container (e.g., "swss", "syncd", "bgp") + + Returns: + bool: True if container was restarted successfully + """ + try: + cmd = f"docker restart {container_name}" + result = dut.shell(cmd, module_ignore_errors=True) + + if result["rc"] == 0: + logger.info(f"Docker container '{container_name}' restarted successfully") + time.sleep(5) # Wait for container to stabilize + return True + else: + logger.error(f"Failed to restart container '{container_name}': {result['stderr']}") + return False + + except Exception as e: + logger.error(f"Error restarting Docker container: {e}") + return False + + +def wait_for_condition(dut, condition_func, timeout=60, interval=2, condition_name="condition"): + """ + Wait for a condition to be true. + + Args: + dut: DUT host object + condition_func: Function that returns True when condition is met + timeout: Maximum time to wait in seconds + interval: Time between checks in seconds + condition_name: Name of condition for logging + + Returns: + bool: True if condition was met within timeout + """ + try: + start_time = datetime.now() + while (datetime.now() - start_time).total_seconds() < timeout: + if condition_func(): + logger.info(f"Condition '{condition_name}' met") + return True + time.sleep(interval) + + logger.warning(f"Timeout waiting for '{condition_name}' ({timeout}s)") + return False + + except Exception as e: + logger.error(f"Error waiting for condition: {e}") + return False + + +# ============================================================================ +# Config CLI Helper Functions +# ============================================================================ + +def get_running_config(dut, interface=None): + """Get running configuration for interface(s).""" + try: + if interface: + cmd = f"show running-configuration interface {interface}" + else: + cmd = "show running-configuration" + + output = dut.shell(cmd, module_ignore_errors=True) + return output.get("stdout", "") + + except Exception as e: + logger.error(f"Error getting running config: {e}") + return "" + + +def save_configuration(dut): + """Save current configuration to startup config.""" + try: + cmd = "config save" + result = dut.shell(cmd, module_ignore_errors=True) + if result["rc"] == 0: + logger.info("Configuration saved") + return True + return False + + except Exception as e: + logger.error(f"Error saving configuration: {e}") + return False + + +def reload_configuration(dut): + """Reload configuration from file.""" + try: + time.sleep(60) # Wait for swSS to be up + cmd = "config reload -y" + result = dut.shell(cmd, module_ignore_errors=True) + if result["rc"] == 0: + logger.info("Configuration reloaded") + time.sleep(60) # Wait for reload to complete + return True + return False + + except Exception as e: + logger.error(f"Error reloading configuration: {e}") + return False diff --git a/tests/link_dampening/test_link_event_damping.py b/tests/link_dampening/test_link_event_damping.py new file mode 100644 index 00000000000..786da45b137 --- /dev/null +++ b/tests/link_dampening/test_link_event_damping.py @@ -0,0 +1,1403 @@ +import logging +import pytest +import time +import json +from datetime import datetime, timedelta + +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until +from tests.link_dampening.link_event_damping_utils import ( + get_dut_fronface_ports, + configure_link_damping, + verify_configuration, + get_link_damping_stats, + clear_link_damping_stats, + generate_link_flap, + get_interface_operational_state, + get_interface_physical_state, + get_redis_db_entries, + validate_redis_persistence, + get_dampening_penalties, + verify_counter_values, + calculate_expected_suppression_time, + check_suppression_active, + inject_traffic_and_verify, + restart_docker_container, + wait_for_condition +) + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.disable_loganalyzer, # disable automatic loganalyzer + pytest.mark.topology('t0', 't1') # Compatible with T0 and T1 topologies +] + +DAMPING_CONFIG_PARAMS = { + "suppress_threshold": 1600, + "reuse_threshold": 1200, + "decay_half_life": 15000, # milliseconds + "max_suppress_time": 30000, # milliseconds + "flap_penalty": 1000 +} + +UNSUPPORTED_CONFIG_PARAMS = { + "suppress_threshold": 1600, + "reuse_threshold": 1200, + "decay_half_life": 45000, # milliseconds - Greater than max_suppress_time + "max_suppress_time": 30000, # milliseconds + "flap_penalty": 1000 +} + +TIMELINE_EVENTS = [ + {"time": 3, "event": "DOWN", "propagated": True}, + {"time": 7, "event": "UP", "propagated": True}, + {"time": 10, "event": "DOWN", "propagated": True}, + {"time": 14, "event": "UP", "propagated": False}, + {"time": 17, "event": "DOWN", "propagated": False}, + {"time": 20, "event": "UP", "propagated": False}, + {"time": 31, "event": "None", "propagated": False}, + {"time": 40, "event": "DOWN", "propagated": True}, + {"time": 44, "event": "UP", "propagated": False}, + {"time": 46, "event": "DOWN", "propagated": False}, + {"time": 61, "event": "None", "propagated": False}, + {"time": 70, "event": "UP", "propagated": True}, + {"time": 100, "event": "DOWN", "propagated": True}, + {"time": 102, "event": "UP", "propagated": True}, + {"time": 105, "event": "DOWN", "propagated": True}, + {"time": 124, "event": "UP", "propagated": False}, + {"time": 152, "event": "None", "propagated": True}, +] + + +class TestLinkEventDampingBasics: + """Test cases for basic link event damping functionality (TC01-TC03)""" + + @pytest.fixture(autouse=True) + def setup_teardown(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo): + """Setup and teardown for each test""" + yield + # Cleanup: clear damping config after each test + # clear_link_damping_stats(duthost) + + def test_tc01_1_normal_link_flap_event_propagation(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC01.1 - Normal Link Flap Event Propagation + + Verify that link up/down events propagate normally when damping is inactive. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + # Get a test interface + test_intf = get_test_interface(dut) + logger.info(f"Using interface {test_intf} for test") + + # Ensure damping is disabled + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + time.sleep(5) + configure_link_damping(dut, test_intf, disabled=True) + + time.sleep(5) + # Generate link UP/DOWN events + initial_state = get_interface_physical_state(dut, test_intf) + logger.info(f"Initial state: {initial_state}") + + generate_link_flap(dut, test_intf, num_flaps=5, interval=1) + + # Verify all physical link changes are propagated + stats = get_link_damping_stats(dut, test_intf) + pre_damping_transitions = int(stats.get('pre_damping_link_transitions', 0)) + logger.warning(f"Pre-damping transitions: {pre_damping_transitions}") + + pytest_assert(pre_damping_transitions > 0, "Expected link transitions to be propagated") + # pytest_assert(pre_damping_transitions == 0, "Expected zero link transitions to be propagated") + + # Operational state should track physical state + op_state = get_interface_operational_state(dut, test_intf) + phys_state = get_interface_physical_state(dut, test_intf) + pytest_assert(op_state == phys_state, + f"Operational state {op_state} should match physical state {phys_state}") + + def test_tc01_2_multiple_sequential_flaps(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC01.2 - Multiple Sequential Flaps + + Verify link flaps are properly tracked and reported. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + configure_link_damping(dut, test_intf, disabled=True) + + time.sleep(10) + + #get stats before the link flap + stats1 = get_link_damping_stats(dut, test_intf) + pre_damping_downs1 = int(stats1.get('pre_damping_down_events', 0)) + pre_damping_ups1 = int(stats1.get('pre_damping_up_events', 0)) + + # Generate 10 sequential flaps + num_flaps = 10 + generate_link_flap(dut, test_intf, num_flaps=num_flaps, interval=0.5) + + time.sleep(20) + # Verify flaps are counted + stats = get_link_damping_stats(dut, test_intf) + pre_damping_downs = int(stats.get('pre_damping_down_events', 0)) + pre_damping_ups = int(stats.get('pre_damping_up_events', 0)) + + logger.warning(f"Pre-damping DOWN events1: {pre_damping_downs1}") + logger.warning(f"Pre-damping UP events1: {pre_damping_ups1}") + logger.warning(f"Pre-damping DOWN events: {pre_damping_downs}") + logger.warning(f"Pre-damping UP events: {pre_damping_ups}") + + pytest_assert(pre_damping_downs >= pre_damping_downs1, + "Expected DOWN events to be recorded") + pytest_assert(pre_damping_ups >= pre_damping_ups1, + "Expected UP events to be recorded") + + def test_tc01_3_simultaneous_flaps_on_multiple_ports(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC01.3 - Simultaneous Flaps on Multiple Ports + + Verify simultaneous link flaps on multiple ports are handled correctly. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + # Get multiple test interfaces + test_intfs = get_test_interfaces(dut, num_intfs=3) + logger.info(f"Using interfaces {test_intfs} for test") + + # Disable damping on all interfaces + for intf in test_intfs: + configure_link_damping(dut, intf, **DAMPING_CONFIG_PARAMS) + time.sleep(5) + configure_link_damping(dut, intf, disabled=True) + + time.sleep(5) + + # Generate flaps on all interfaces simultaneously + for intf in test_intfs: + generate_link_flap(dut, intf, num_flaps=3, interval=0.5) + + time.sleep(5) + # Verify all interfaces have reported events + for intf in test_intfs: + stats = get_link_damping_stats(dut, intf) + pre_damping_transitions = int(stats.get('pre_damping_link_transitions', 0)) + logger.warning(f"Pre-damping DOWN events: {pre_damping_transitions}") + pytest_assert(pre_damping_transitions > 0, + f"Expected transitions on {intf}") + + +class TestLinkEventDampingConfiguration: + """Test cases for damping configuration validation (TC02-TC03)""" + + def test_tc02_1_basic_link_damping_configuration(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.1 - Basic Link Damping Configuration + + Verify that damping configuration is applied correctly. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Configure damping with all parameters + configure_link_damping(dut, test_intf, + suppress_threshold=DAMPING_CONFIG_PARAMS["suppress_threshold"], + reuse_threshold=DAMPING_CONFIG_PARAMS["reuse_threshold"], + decay_half_life=DAMPING_CONFIG_PARAMS["decay_half_life"], + max_suppress_time=DAMPING_CONFIG_PARAMS["max_suppress_time"], + flap_penalty=DAMPING_CONFIG_PARAMS["flap_penalty"]) + + time.sleep(5) + # Verify configuration in CONFIG_DB + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, "Configuration parameters not found in CONFIG_DB") + + logger.info(f"Damping configuration applied successfully on {test_intf}") + + def test_tc02_2_config_db_persistence(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.2 - CONFIG_DB Persistence + + Verify damping configuration persists in CONFIG_DB. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + time.sleep(5) + # Query CONFIG_DB + config_entries = get_redis_db_entries(dut, "CONFIG_DB", f"*LINK_EVENT_DAMPING*") + pytest_assert(config_entries, f"No CONFIG_DB entries found for {test_intf}") + + logger.info(f"CONFIG_DB entries: {config_entries}") + + def test_tc02_3_redis_persistence(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.3 - Redis Persistence + + Verify damping configuration persists in Redis databases. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + time.sleep(5) + # Verify Redis persistence + is_persistent = validate_redis_persistence(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_persistent, "Configuration not persisted in Redis") + + def test_tc02_4_multiple_configuration_profiles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.4 - Multiple Configuration Profiles + + Verify different configuration profiles can be applied to different interfaces. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + + # Apply different configurations to different interfaces + config1 = {"suppress_threshold": 1600, "max_suppress_time": 30000} # milliseconds + config2 = {"suppress_threshold": 800, "max_suppress_time": 20000} # milliseconds + + configure_link_damping(dut, test_intfs[0], **config1) + configure_link_damping(dut, test_intfs[1], **config2) + + time.sleep(5) + # Verify both configurations are independent + verify_configuration(dut, test_intfs[0], config1) + verify_configuration(dut, test_intfs[1], config2) + + logger.info("Multiple configuration profiles verified") + + def test_tc02_5_individual_parameter_validation(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.5 - Individual Parameter Validation + + Verify each damping parameter is individually configurable. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Test each parameter individually + params_to_test = [ + {"suppress_threshold": 2000}, + {"reuse_threshold": 1500}, + {"decay_half_life": 20000}, # milliseconds + {"max_suppress_time": 45000}, # milliseconds + {"flap_penalty": 600} + ] + + for param in params_to_test: + configure_link_damping(dut, test_intf, **param) + time.sleep(5) + is_configured = verify_configuration(dut, test_intf, param) + pytest_assert(is_configured, f"Parameter {param} not configured correctly") + + def test_tc02_6_configuration_synchronization(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC02.6 - Configuration Synchronization + + Verify configuration is synchronized across all layers. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + time.sleep(5) + # Verify in CONFIG_DB + config_db_ok = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + + # Verify in Redis databases + redis_ok = validate_redis_persistence(dut, test_intf, DAMPING_CONFIG_PARAMS) + + pytest_assert(config_db_ok and redis_ok, "Configuration not synchronized") + + +class TestLinkEventDampingUnsupported: + """Test cases for unsupported configuration handling (TC03)""" + + def test_tc03_1_decay_exceeds_max_suppress_time(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.1 - Decay Exceeds Max Suppress Time + + Verify damping is disabled when decay-half-life > max-suppress-time. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Apply unsupported configuration + configure_link_damping(dut, test_intf, **UNSUPPORTED_CONFIG_PARAMS) + + time.sleep(5) + clear_link_damping_stats(dut) + # Generate link flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + time.sleep(5) + # Verify damping is disabled - all events should propagate + stats = get_link_damping_stats(dut, test_intf) + post_damping_propagated = int(stats.get('post_damping_propagated_transitions', 0)) + pre_damping_transitions = int(stats.get('pre_damping_link_transitions', 0)) + + logger.warning(f"Pre-damping transitions: {pre_damping_transitions}") + logger.warning(f"Post-damping propagated: {post_damping_propagated}") + + # All events should be propagated (damping disabled) + pytest_assert(post_damping_propagated == pre_damping_transitions or post_damping_propagated >= pre_damping_transitions - 1, + "Unsupported config should disable damping") + + def test_tc03_2_zero_flap_penalty(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.2 - Zero Flap Penalty + + Verify configuration with zero flap penalty is handled. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Zero penalty configuration + zero_penalty_config = DAMPING_CONFIG_PARAMS.copy() + zero_penalty_config["flap_penalty"] = 0 + + configure_link_damping(dut, test_intf, **zero_penalty_config) + + # Generate flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Verify configuration accepted + is_configured = verify_configuration(dut, test_intf, {"flap_penalty": 0}) + pytest_assert(is_configured, "Zero penalty configuration should be accepted") + + def test_tc03_3_suppress_less_than_reuse(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.3 - Suppress Less Than Reuse + + Verify invalid configuration (suppress < reuse) is handled. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Invalid configuration + invalid_config = DAMPING_CONFIG_PARAMS.copy() + invalid_config["suppress_threshold"] = 800 + invalid_config["reuse_threshold"] = 1000 + + # This should either be rejected or handled gracefully + configure_link_damping(dut, test_intf, **invalid_config) + + logger.info("Invalid configuration handled") + + def test_tc03_4_zero_reuse_threshold(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.4 - Zero Reuse Threshold + + Verify configuration with zero reuse threshold is handled. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + zero_reuse_config = DAMPING_CONFIG_PARAMS.copy() + zero_reuse_config["reuse_threshold"] = 0 + + configure_link_damping(dut, test_intf, **zero_reuse_config) + + logger.info("Zero reuse threshold configuration handled") + + def test_tc03_5_zero_max_suppress_time(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.6 - Zero Max Suppress Time + + Verify configuration with zero max suppress time is handled. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + zero_suppress_config = DAMPING_CONFIG_PARAMS.copy() + zero_suppress_config["max_suppress_time"] = 0 + + configure_link_damping(dut, test_intf, **zero_suppress_config) + + logger.info("Zero max suppress time configuration handled") + + def test_tc03_6_error_logging(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC03.9 - Error Logging + + Verify error messages are logged for invalid configurations. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Apply invalid configuration and check for error logs + configure_link_damping(dut, test_intf, suppress_threshold=100, max_suppress_time=10000) # milliseconds + + # Check system logs for errors + output = dut.shell("tail -100 /var/log/syslog | grep -i 'damping\\|error' || echo 'no errors'") + logger.info(f"System logs: {output['stdout']}") + + +class TestLinkEventDampingMixedConfig: + """Test cases for mixed damping configuration (TC04)""" + + def test_tc04_1_basic_mixed_configuration(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.1 - Basic Mixed Configuration + + Verify damping on some ports and no damping on others. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + damped_intf = test_intfs[0] + undamped_intf = test_intfs[1] + + # Configure damping on first interface + configure_link_damping(dut, damped_intf, **DAMPING_CONFIG_PARAMS) + + # Disable damping on second interface + configure_link_damping(dut, undamped_intf, disabled=True) + + # Verify both configurations + assert verify_configuration(dut, damped_intf, DAMPING_CONFIG_PARAMS) + + logger.info("Mixed configuration applied successfully") + + def test_tc04_2_simultaneous_flaps_damped_vs_undamped(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.2 - Simultaneous Flaps (Damped vs Undamped) + + Compare behavior of damped and undamped ports. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + damped_intf = test_intfs[0] + undamped_intf = test_intfs[1] + + configure_link_damping(dut, damped_intf, **DAMPING_CONFIG_PARAMS) + configure_link_damping(dut, undamped_intf, disabled=True) + + clear_link_damping_stats(dut) + + # Generate identical flap patterns + num_flaps = 10 + for _ in range(num_flaps): + generate_link_flap(dut, damped_intf, num_flaps=1, interval=0.5) + generate_link_flap(dut, undamped_intf, num_flaps=1, interval=0.5) + + # Get stats + damped_stats = get_link_damping_stats(dut, damped_intf) + undamped_stats = get_link_damping_stats(dut, undamped_intf) + + damped_propagated = int(damped_stats.get('post_damping_propagated_transitions', 0)) + undamped_propagated = int(undamped_stats.get('post_damping_propagated_transitions', 0)) + + logger.info(f"Damped interface propagated: {damped_propagated}") + logger.info(f"Undamped interface propagated: {undamped_propagated}") + + # Undamped should propagate more events + pytest_assert(undamped_propagated >= damped_propagated, + "Undamped interface should propagate more events") + + def test_tc04_3_different_damping_profiles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.3 - Different Damping Profiles + + Verify different damping profiles on different ports. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + + config1 = {"suppress_threshold": 1600, "max_suppress_time": 30000} # milliseconds + config2 = {"suppress_threshold": 800, "max_suppress_time": 20000} # milliseconds + + configure_link_damping(dut, test_intfs[0], **config1) + configure_link_damping(dut, test_intfs[1], **config2) + + logger.info("Different damping profiles applied") + + def test_tc04_4_port_independence(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.4 - Port Independence + + Verify damping on one port doesn't affect others. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=3) + + # Configure damping on one port + configure_link_damping(dut, test_intfs[0], **DAMPING_CONFIG_PARAMS) + + # Generate flaps on first port + generate_link_flap(dut, test_intfs[0], num_flaps=10, interval=0.5) + + # Verify other ports are unaffected + for intf in test_intfs[1:]: + phys_state = get_interface_physical_state(dut, intf) + op_state = get_interface_operational_state(dut, intf) + # States might differ due to the flaps on other port, but shouldn't be suppressed + logger.info(f"Interface {intf} state: physical={phys_state}, operational={op_state}") + + def test_tc04_5_flap_pattern_comparison(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.6 - Flap Pattern Comparison + + Compare flap patterns between damped and undamped ports. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + + configure_link_damping(dut, test_intfs[0], **DAMPING_CONFIG_PARAMS) + configure_link_damping(dut, test_intfs[1], disabled=True) + + logger.info("Flap pattern comparison set up") + + def test_tc04_6_large_scale_mixed_configuration(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC04.7 - Large-Scale Mixed Configuration + + Verify system handles large-scale mixed configurations. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=10) + + # Apply alternating configurations + for i, intf in enumerate(test_intfs): + if i % 2 == 0: + configure_link_damping(dut, intf, **DAMPING_CONFIG_PARAMS) + else: + configure_link_damping(dut, intf, disabled=True) + + logger.info(f"Large-scale mixed configuration applied to {len(test_intfs)} interfaces") + + +class TestLinkEventDampingOperationalState: + """Test cases for operational state accuracy (TC05)""" + + def test_tc05_1_operational_state_frozen_during_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC05.1 - Operational State Frozen During Suppression + + Verify operational state remains frozen while damping is active. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate flaps to trigger damping + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check if suppression is active + if check_suppression_active(dut, test_intf): + op_state_during = get_interface_operational_state(dut, test_intf) + phys_state = get_interface_physical_state(dut, test_intf) + + logger.info(f"During suppression - Op state: {op_state_during}, Phys state: {phys_state}") + pytest_assert(op_state_during != phys_state or op_state_during == "down", + "Operational state should be frozen during suppression") + + def test_tc05_2_operational_state_updates_after_suppression_ends(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC05.2 - Operational State Updates After Suppression Ends + + Verify operational state updates after suppression ends. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Wait for suppression to end (max_suppress_time + buffer) + max_suppress_ms = DAMPING_CONFIG_PARAMS.get("max_suppress_time", 30000) # milliseconds + wait_time = (max_suppress_ms / 1000) + 10 # convert to seconds and add buffer + logger.info(f"Waiting {wait_time} seconds for suppression to end") + time.sleep(wait_time) + + # Verify operational state matches physical state + op_state = get_interface_operational_state(dut, test_intf) + phys_state = get_interface_physical_state(dut, test_intf) + + logger.info(f"After suppression - Op state: {op_state}, Phys state: {phys_state}") + pytest_assert(op_state == phys_state, + "Operational state should match physical state after suppression ends") + + def test_tc05_3_physical_vs_operational_state_divergence(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC05.3 - Physical vs Operational State Divergence During Suppression + + Verify physical and operational states diverge during suppression. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Record initial states + initial_phys = get_interface_physical_state(dut, test_intf) + initial_op = get_interface_operational_state(dut, test_intf) + + # Generate flaps to trigger suppression + generate_link_flap(dut, test_intf, num_flaps=10, interval=0.3) + + # Check for divergence + if check_suppression_active(dut, test_intf): + phys_state = get_interface_physical_state(dut, test_intf) + op_state = get_interface_operational_state(dut, test_intf) + + logger.info(f"Physical state: {phys_state}, Operational state: {op_state}") + # States should diverge if suppression is active + + def test_tc05_4_penalty_decay_and_state_recovery(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC05.4 - Penalty Decay and State Recovery + + Verify state recovers as penalty decays. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Monitor penalty decay + initial_penalty = get_dampening_penalties(dut, test_intf) + logger.info(f"Initial penalty: {initial_penalty}") + + # Wait and check penalty decay + time.sleep(10) + current_penalty = get_dampening_penalties(dut, test_intf) + logger.info(f"Current penalty after 10s: {current_penalty}") + + # Penalty should decay + pytest_assert(current_penalty < initial_penalty or current_penalty == 0, + "Penalty should decay over time") + + def test_tc05_5_multiple_suppression_cycles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC05.5 - Multiple Suppression Cycles + + Verify system handles multiple suppression cycles correctly. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Cycle 1: Generate flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + time.sleep(35) + + # Cycle 2: Generate more flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + time.sleep(35) + + logger.info("Multiple suppression cycles completed") + +class TestLinkEventDampingFrequency: + """Test cases for flap frequency effects (TC06)""" + + def test_tc06_1_frequent_flaps_longer_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.1 - Frequent Flaps Longer Suppression + + Verify frequent flaps result in longer suppression. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + time.sleep(3) + # Generate frequent flaps + start_time = datetime.now() + generate_link_flap(dut, test_intf, num_flaps=10, interval=6) + + # Check suppression duration + suppression_start = datetime.now() + while check_suppression_active(dut, test_intf) and (datetime.now() - suppression_start).seconds < 60: + time.sleep(2) + + suppression_duration = (datetime.now() - suppression_start).seconds + logger.warning(f"Suppression duration: {suppression_duration} seconds") + + # Should be longer than minimal + pytest_assert(suppression_duration > 5, "Suppression should last a reasonable time") + + def test_tc06_2_infrequent_flaps_shorter_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.2 - Infrequent Flaps Shorter Suppression + + Verify infrequent flaps result in shorter suppression. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate infrequent flaps (sparse) + generate_link_flap(dut, test_intf, num_flaps=2, interval=5) + + logger.info("Infrequent flaps generated") + + def test_tc06_3_penalty_accumulation_difference(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.3 - Penalty Accumulation Difference + + Verify different penalty accumulation for different flap frequencies. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + + for intf in test_intfs: + configure_link_damping(dut, intf, **DAMPING_CONFIG_PARAMS) + + # Frequent flaps on first interface + generate_link_flap(dut, test_intfs[0], num_flaps=10, interval=0.3) + + # Infrequent flaps on second interface + generate_link_flap(dut, test_intfs[1], num_flaps=2, interval=5) + + logger.info("Penalty accumulation difference verified") + + def test_tc06_4_decay_rate_same_for_both(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.4 - Decay Rate Same for Both + + Verify decay rate is consistent regardless of frequency. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Record penalty at different times + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + penalty_t0 = get_dampening_penalties(dut, test_intf) + time.sleep(5) + penalty_t5 = get_dampening_penalties(dut, test_intf) + time.sleep(5) + penalty_t10 = get_dampening_penalties(dut, test_intf) + + logger.info(f"Penalty at t=0: {penalty_t0}, t=5: {penalty_t5}, t=10: {penalty_t10}") + + def test_tc06_5_recovery_time_proportional_to_frequency(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.5 - Recovery Time Proportional to Frequency + + Verify recovery time varies with flap frequency. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate flaps and measure recovery + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + logger.info("Recovery time measurement set up") + + def test_tc06_6_mixed_pattern_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.6 - Mixed Pattern Suppression + + Verify suppression behavior with mixed flap patterns. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate mixed pattern: frequent, pause, sparse + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.3) + time.sleep(5) + generate_link_flap(dut, test_intf, num_flaps=2, interval=5) + + logger.info("Mixed pattern suppression tested") + + def test_tc06_7_threshold_crossing_different_timing(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC06.7 - Threshold Crossing Different Timing + + Verify different timing for threshold crossing. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=2) + + config_aggressive = DAMPING_CONFIG_PARAMS.copy() + config_conservative = DAMPING_CONFIG_PARAMS.copy() + config_conservative["suppress_threshold"] = 3200 + + configure_link_damping(dut, test_intfs[0], **config_aggressive) + configure_link_damping(dut, test_intfs[1], **config_conservative) + + logger.info("Threshold crossing timing test set up") + + +class TestLinkEventDampingCounters: + """Test cases for counter verification (TC07)""" + + def test_tc07_1_pre_damping_link_transitions_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.1 - Pre-Damping Link Transitions Counter + + Verify pre-damping link transitions counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate flaps + num_flaps = 5 + generate_link_flap(dut, test_intf, num_flaps=num_flaps, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + transitions = int(stats.get('pre_damping_link_transitions', 0)) + + logger.info(f"Pre-damping link transitions: {transitions}") + pytest_assert(transitions > 0, "Pre-damping transitions should be recorded") + + def test_tc07_2_post_damping_propagated_transitions_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.2 - Post-Damping Propagated Transitions Counter + + Verify post-damping propagated transitions counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + propagated = int(stats.get('post_damping_propagated_transitions', 0)) + + logger.info(f"Post-damping propagated transitions: {propagated}") + + def test_tc07_3_pre_damping_up_events_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.3 - Pre-Damping UP Events Counter + + Verify pre-damping UP events counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate UP events + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + up_events = int(stats.get('pre_damping_up_events', 0)) + + logger.info(f"Pre-damping UP events: {up_events}") + pytest_assert(up_events > 0, "UP events should be recorded") + + def test_tc07_4_pre_damping_down_events_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.4 - Pre-Damping DOWN Events Counter + + Verify pre-damping DOWN events counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate DOWN events + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + down_events = int(stats.get('pre_damping_down_events', 0)) + + logger.info(f"Pre-damping DOWN events: {down_events}") + pytest_assert(down_events > 0, "DOWN events should be recorded") + + def test_tc07_5_post_damping_up_advertised_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.5 - Post-Damping UP Advertised Counter + + Verify post-damping UP advertised counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate UP events + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + up_advertised = int(stats.get('post_damping_up_advertised', 0)) + + logger.info(f"Post-damping UP advertised: {up_advertised}") + + def test_tc07_6_post_damping_down_advertised_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.6 - Post-Damping DOWN Advertised Counter + + Verify post-damping DOWN advertised counter. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate DOWN events + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Check counter + stats = get_link_damping_stats(dut, test_intf) + down_advertised = int(stats.get('post_damping_down_advertised', 0)) + + logger.info(f"Post-damping DOWN advertised: {down_advertised}") + + def test_tc07_7_counter_consistency_across_cycles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.7 - Counter Consistency Across Cycles + + Verify counters remain consistent across multiple cycles. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # First cycle + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + stats1 = get_link_damping_stats(dut, test_intf) + count1 = int(stats1.get('pre_damping_link_transitions', 0)) + + # Second cycle + generate_link_flap(dut, test_intf, num_flaps=3, interval=0.5) + stats2 = get_link_damping_stats(dut, test_intf) + count2 = int(stats2.get('pre_damping_link_transitions', 0)) + + logger.info(f"Cycle 1 transitions: {count1}, Cycle 2 transitions: {count2}") + pytest_assert(count2 >= count1, "Counter should monotonically increase") + + def test_tc07_8_counter_increments_proportional_to_events(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.8 - Counter Increments Proportional to Events + + Verify counter increments are proportional to events. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate different numbers of flaps + generate_link_flap(dut, test_intf, num_flaps=10, interval=0.5) + + stats = get_link_damping_stats(dut, test_intf) + transitions = int(stats.get('pre_damping_link_transitions', 0)) + + logger.info(f"Transitions for 10 flaps: {transitions}") + pytest_assert(transitions >= 10, "Counter should reflect number of events") + + def test_tc07_9_suppressed_events_not_in_post_damping(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.9 - Suppressed Events Not in Post-Damping + + Verify suppressed events are not counted in post-damping. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate flaps that trigger suppression + generate_link_flap(dut, test_intf, num_flaps=20, interval=0.3) + + stats = get_link_damping_stats(dut, test_intf) + pre_transitions = int(stats.get('pre_damping_link_transitions', 0)) + post_transitions = int(stats.get('post_damping_propagated_transitions', 0)) + + logger.info(f"Pre-damping: {pre_transitions}, Post-damping: {post_transitions}") + pytest_assert(post_transitions <= pre_transitions, + "Post-damping should not include suppressed events") + + def test_tc07_10_counter_reset_and_recovery(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC07.10 - Counter Reset and Recovery + + Verify counters can be reset and recovery is tracked. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate initial flaps + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + # Clear stats + clear_link_damping_stats(dut) + + stats = get_link_damping_stats(dut, test_intf) + transitions = int(stats.get('pre_damping_link_transitions', 0)) + + logger.info(f"Transitions after reset: {transitions}") + pytest_assert(transitions == 0, "Counters should reset to zero") + + +class TestLinkEventDampingTimeline: + """Test cases for timeline validation (TC09)""" + + def test_tc09_1_timeline_event_sequence_execution(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC09.1 - Timeline Event Sequence Execution + + Execute deterministic timeline of events. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + logger.info(f"Starting timeline test on {test_intf}") + logger.info(f"Configuration: {DAMPING_CONFIG_PARAMS}") + + clear_link_damping_stats(dut) + + # Execute timeline events + start_time = datetime.now() + execution_log = [] + + for event in TIMELINE_EVENTS: + if event["event"] != "None": + # Wait until event time + event_time = event["time"] + wait_duration = event_time - (datetime.now() - start_time).total_seconds() + + if wait_duration > 0: + time.sleep(wait_duration) + + # Execute event + logger.info(f"Executing event at t={event['time']}: {event['event']}") + generate_link_flap(dut, test_intf, num_flaps=1, interval=0) + + execution_log.append({ + "time": event["time"], + "event": event["event"], + "expected_propagated": event["propagated"] + }) + + + +class TestLinkEventDampingPersistence: + """Test cases for persistence across reboots and docker restarts (TC10)""" + + def test_tc10_1_damping_config_persists_after_reboot(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.1 - Damping Config Persists After Reboot + + Verify damping configuration persists after device reboot. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Configure damping before reboot + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Verify configuration before reboot + is_configured_before = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured_before, "Configuration should exist before reboot") + + logger.info(f"Rebooting {dut.hostname}...") + dut.reboot() + + time.sleep(60) + # Verify configuration after reboot + is_configured_after = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured_after, "Configuration should persist after reboot") + + logger.info("Configuration persisted after reboot") + + def test_tc10_2_damping_functionality_after_reboot(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.2 - Damping Functionality After Reboot + + Verify damping functionality works correctly after reboot. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + + # Configure and reboot + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + logger.info("Rebooting...") + dut.reboot() + + time.sleep(60) + clear_link_damping_stats(dut) + + # Generate flaps after reboot + generate_link_flap(dut, test_intf, num_flaps=10, interval=0.5) + + # Verify damping is working + stats = get_link_damping_stats(dut, test_intf) + pre_damping = int(stats.get('pre_damping_link_transitions', 0)) + post_damping = int(stats.get('post_damping_propagated_transitions', 0)) + + logger.info(f"Pre-damping: {pre_damping}, Post-damping: {post_damping}") + pytest_assert(post_damping <= pre_damping, "Damping should work after reboot") + + def test_tc10_3_counters_preserved_after_reboot(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.3 - Counters Preserved After Reboot + + Verify counters are preserved across reboot. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + clear_link_damping_stats(dut) + + # Generate events before reboot + generate_link_flap(dut, test_intf, num_flaps=5, interval=0.5) + + stats_before = get_link_damping_stats(dut, test_intf) + transitions_before = int(stats_before.get('pre_damping_link_transitions', 0)) + + logger.info(f"Transitions before reboot: {transitions_before}") + + # Reboot + logger.info("Rebooting...") + dut.reboot() + + time.sleep(60) + # Check counters after reboot + stats_after = get_link_damping_stats(dut, test_intf) + transitions_after = int(stats_after.get('pre_damping_link_transitions', 0)) + + logger.info(f"Transitions after reboot: {transitions_after}") + + def test_tc10_4_multiple_reboot_cycles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.4 - Multiple Reboot Cycles + + Verify system survives multiple reboot cycles. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Multiple reboot cycles + num_reboots = 2 + for cycle in range(num_reboots): + logger.info(f"Reboot cycle {cycle + 1}/{num_reboots}") + dut.reboot() + + time.sleep(60) + # Verify configuration + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, f"Config lost after reboot cycle {cycle + 1}") + + logger.info("Multiple reboot cycles completed successfully") + + + def test_tc10_5_concurrent_damping_multiple_ports(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.5 - Concurrent Damping Multiple Ports + + Verify concurrent damping on multiple ports survives reboot. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intfs = get_test_interfaces(dut, num_intfs=5) + + # Configure all interfaces + for intf in test_intfs: + configure_link_damping(dut, intf, **DAMPING_CONFIG_PARAMS) + + # Reboot + logger.info("Rebooting...") + dut.reboot() + + time.sleep(60) + # Verify all configurations + for intf in test_intfs: + is_configured = verify_configuration(dut, intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, f"Config lost for {intf} after reboot") + + logger.info("Concurrent damping on multiple ports verified after reboot") + + def test_tc10_6_reboot_during_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.6 - Reboot During Suppression + + Verify system handles reboot while suppression is active. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Generate flaps to trigger suppression + generate_link_flap(dut, test_intf, num_flaps=10, interval=0.3) + + # Verify suppression is active + if check_suppression_active(dut, test_intf): + logger.info("Suppression is active, rebooting...") + dut.reboot() + + time.sleep(60) + # Verify system recovered + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, "Config should persist after reboot during suppression") + time.sleep(120) + + def test_tc10_7_bgp_docker_restart(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.7 - BGP Docker Restart + + Verify damping persists after BGP docker restart. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Restart BGP container + try: + restart_docker_container(dut, "bgp") + logger.info("BGP docker restarted") + + # Verify config persists + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, "Config should persist after BGP docker restart") + time.sleep(120) + except Exception as e: + logger.warning(f"BGP docker restart not available: {e}") + + def test_tc10_8_swss_docker_restart(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.8 - SWSS Docker Restart + + Verify damping persists after SWSS docker restart. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Restart SWSS container + try: + restart_docker_container(dut, "swss") + logger.info("SWSS docker restarted") + + time.sleep(60) + # Verify config persists + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, "Config should persist after SWSS docker restart") + time.sleep(120) + except Exception as e: + logger.warning(f"SWSS docker restart failed: {e}") + + def test_tc10_9_syncd_docker_restart(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """TC10.9 - Syncd Docker Restart + + Verify damping persists after Syncd docker restart. + """ + logger = logging.getLogger(__name__) + dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + time.sleep(120) + test_intf = get_test_interface(dut) + configure_link_damping(dut, test_intf, **DAMPING_CONFIG_PARAMS) + + # Restart Syncd container + try: + restart_docker_container(dut, "syncd") + logger.info("Syncd docker restarted") + + # Verify config persists + is_configured = verify_configuration(dut, test_intf, DAMPING_CONFIG_PARAMS) + pytest_assert(is_configured, "Config should persist after Syncd docker restart") + time.sleep(120) + except Exception as e: + logger.warning(f"Syncd docker restart failed: {e}") + + +# ============================================================================ +# Helper Functions +# ============================================================================ + +def get_test_interface(dut): + """Get a test interface from the DUT""" + interfaces = dut.show_and_parse("show interfaces status") + if interfaces: + return interfaces[0]["interface"] + pytest_assert(False, "No interfaces found on DUT") + + +def get_test_interfaces(dut, num_intfs=2): + """Get multiple test interfaces from the DUT""" + interfaces = dut.show_and_parse("show interfaces status") + pytest_assert(len(interfaces) >= num_intfs, f"Need at least {num_intfs} interfaces, found {len(interfaces)}") + return [intf["interface"] for intf in interfaces[:num_intfs]] From b0b6ff400f8f5aa53817f8938b1a17a3d193758b Mon Sep 17 00:00:00 2001 From: Muneer CH Date: Thu, 11 Jun 2026 05:15:28 +0000 Subject: [PATCH 2/5] Addressed review comments --- tests/link_dampening/conftest.py | 1 - tests/link_dampening/link_event_damping_utils.py | 4 +--- tests/link_dampening/test_link_event_damping.py | 14 ++++++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/link_dampening/conftest.py b/tests/link_dampening/conftest.py index a54d3bfcf5a..ac99f60712b 100644 --- a/tests/link_dampening/conftest.py +++ b/tests/link_dampening/conftest.py @@ -5,7 +5,6 @@ import logging import pytest -#from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.assertions import pytest_assert as pt_assert from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports diff --git a/tests/link_dampening/link_event_damping_utils.py b/tests/link_dampening/link_event_damping_utils.py index 4042b81cd89..be3db72a051 100644 --- a/tests/link_dampening/link_event_damping_utils.py +++ b/tests/link_dampening/link_event_damping_utils.py @@ -10,8 +10,6 @@ import logging import time -import json -import re from datetime import datetime from natsort import natsorted @@ -66,7 +64,7 @@ def configure_link_damping(dut, interface, suppress_threshold=None, reuse_thresh # TODO: Verify the correct command to disable damping cmd = f"config interface damping algo {interface} disabled" result = dut.shell(cmd, module_ignore_errors=True) - logger.info(f"Link damping disabled on {interface}") + logger.info(f"Link damping disabled on {interface} result {result}") return True # Step 1: Configure the damping algorithm diff --git a/tests/link_dampening/test_link_event_damping.py b/tests/link_dampening/test_link_event_damping.py index 786da45b137..fdbdbb12cae 100644 --- a/tests/link_dampening/test_link_event_damping.py +++ b/tests/link_dampening/test_link_event_damping.py @@ -1,11 +1,9 @@ import logging import pytest import time -import json -from datetime import datetime, timedelta +from datetime import datetime from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until from tests.link_dampening.link_event_damping_utils import ( get_dut_fronface_ports, configure_link_damping, @@ -256,6 +254,8 @@ def test_tc02_3_redis_persistence(self, duthost, duthosts, enum_rand_one_per_hws is_persistent = validate_redis_persistence(dut, test_intf, DAMPING_CONFIG_PARAMS) pytest_assert(is_persistent, "Configuration not persisted in Redis") + logger.info(f"Persistent: {is_persistent}") + def test_tc02_4_multiple_configuration_profiles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC02.4 - Multiple Configuration Profiles @@ -305,6 +305,8 @@ def test_tc02_5_individual_parameter_validation(self, duthost, duthosts, enum_ra is_configured = verify_configuration(dut, test_intf, param) pytest_assert(is_configured, f"Parameter {param} not configured correctly") + logger.info("individual Parameter configuration verified") + def test_tc02_6_configuration_synchronization(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC02.6 - Configuration Synchronization @@ -326,6 +328,7 @@ def test_tc02_6_configuration_synchronization(self, duthost, duthosts, enum_rand pytest_assert(config_db_ok and redis_ok, "Configuration not synchronized") + logger.info("Configuration synchronisation with CONFIG DB verified") class TestLinkEventDampingUnsupported: """Test cases for unsupported configuration handling (TC03)""" @@ -384,6 +387,8 @@ def test_tc03_2_zero_flap_penalty(self, duthost, duthosts, enum_rand_one_per_hws is_configured = verify_configuration(dut, test_intf, {"flap_penalty": 0}) pytest_assert(is_configured, "Zero penalty configuration should be accepted") + logger.info("Zero penalty configuaration verified") + def test_tc03_3_suppress_less_than_reuse(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC03.3 - Suppress Less Than Reuse @@ -672,6 +677,7 @@ def test_tc05_3_physical_vs_operational_state_divergence(self, duthost, duthosts phys_state = get_interface_physical_state(dut, test_intf) op_state = get_interface_operational_state(dut, test_intf) + logger.info(f"Initial Physical state: {initial_phys}, Operational state: {initial_op}") logger.info(f"Physical state: {phys_state}, Operational state: {op_state}") # States should diverge if suppression is active @@ -748,7 +754,7 @@ def test_tc06_1_frequent_flaps_longer_suppression(self, duthost, duthosts, enum_ time.sleep(2) suppression_duration = (datetime.now() - suppression_start).seconds - logger.warning(f"Suppression duration: {suppression_duration} seconds") + logger.warning(f"Suppression start time: {start_time}, duration: {suppression_duration} seconds") # Should be longer than minimal pytest_assert(suppression_duration > 5, "Suppression should last a reasonable time") From ddb8e083829473805d82d0b32adfde5b521ee01f Mon Sep 17 00:00:00 2001 From: Muneer CH Date: Thu, 18 Jun 2026 05:35:04 +0000 Subject: [PATCH 3/5] Addressing review comments Signed-off-by: Muneer Cheruvangot House --- tests/link_dampening/conftest.py | 11 +- .../link_event_damping_utils.py | 30 ++--- .../link_dampening/test_link_event_damping.py | 120 +++++++++++------- 3 files changed, 93 insertions(+), 68 deletions(-) diff --git a/tests/link_dampening/conftest.py b/tests/link_dampening/conftest.py index ac99f60712b..61e0a7b29a4 100644 --- a/tests/link_dampening/conftest.py +++ b/tests/link_dampening/conftest.py @@ -5,9 +5,10 @@ import logging import pytest -from tests.common.helpers.assertions import pytest_assert as pt_assert -from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports +# from tests.common.helpers.assertions import pytest_assert as pt_assert +# from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports +from tests.common.helpers.assertions import pytest_assert from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports logger = logging.getLogger(__name__) @@ -25,7 +26,7 @@ def link_dampening_test_interface(duthosts, enum_rand_one_per_hwsku_frontend_hos # Get all front-facing interfaces front_interfaces = get_dut_fronface_ports(dut, tbinfo) - #pytest_assert(front_interfaces, "No front-facing interfaces found on DUT") + # pytest_assert(front_interfaces, "No front-facing interfaces found on DUT") # Return the first interface return front_interfaces[0] @@ -43,7 +44,7 @@ def link_dampening_test_interfaces(duthosts, enum_rand_one_per_hwsku_frontend_ho # Get all front-facing interfaces front_interfaces = get_dut_fronface_ports(dut, tbinfo) - #pytest_assert(len(front_interfaces) >= 2, "Need at least 2 front-facing interfaces") + # pytest_assert(len(front_interfaces) >= 2, "Need at least 2 front-facing interfaces") # Return up to 5 interfaces return front_interfaces[:5] @@ -61,7 +62,7 @@ def cleanup_link_damping(duthosts, enum_rand_one_per_hwsku_frontend_hostname): dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] # Clear link damping stats dut.shell("redis-cli -n 0 --scan --match 'LINK_DAMPING_STATS:*' | xargs redis-cli -n 0 DEL", - module_ignore_errors=True) + module_ignore_errors=True) logger.info("Cleaned up link damping statistics") except Exception as e: logger.warning(f"Could not cleanup link damping stats: {e}") diff --git a/tests/link_dampening/link_event_damping_utils.py b/tests/link_dampening/link_event_damping_utils.py index be3db72a051..a6d0e56de17 100644 --- a/tests/link_dampening/link_event_damping_utils.py +++ b/tests/link_dampening/link_event_damping_utils.py @@ -39,8 +39,8 @@ def get_dut_fronface_ports(duthost, tbinfo): def configure_link_damping(dut, interface, suppress_threshold=None, reuse_threshold=None, - decay_half_life=None, max_suppress_time=None, flap_penalty=None, - algorithm="aied", disabled=False): + decay_half_life=None, max_suppress_time=None, flap_penalty=None, + algorithm="aied", disabled=False): """ Configure link damping parameters on a specific interface using AIED algorithm. @@ -98,8 +98,8 @@ def configure_link_damping(dut, interface, suppress_threshold=None, reuse_thresh if result["rc"] == 0: logger.info(f"Link damping parameters configured on {interface}: " - f"threshold={suppress_threshold}, reuse={reuse_threshold}, " - f"decay={decay_half_life}ms, max_suppress={max_suppress_time}ms, penalty={flap_penalty}") + f"threshold={suppress_threshold}, reuse={reuse_threshold}, " + f"decay={decay_half_life}ms, max_suppress={max_suppress_time}ms, penalty={flap_penalty}") return True else: logger.warning(f"Failed to configure damping parameters on {interface}: {result.get('stderr', '')}") @@ -172,7 +172,7 @@ def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, n logger.info(f"Generating {num_flaps} link flaps on {dut_interface} with {interval}s interval") for flap_num in range(num_flaps): - # Admin down + # Admin dow Exception as en logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Admin down on {dut_interface}") dut.shell(f"config interface shutdown {dut_interface}", module_ignore_errors=True) time.sleep(interval / 2) @@ -187,19 +187,19 @@ def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, n else: # Use fanout switch interface (preferred method) logger.info(f"Generating {num_flaps} link flaps on {dut_interface} " - f"(fanout {fanout_interface}) with {interval}s interval") + f"(fanout {fanout_interface}) with {interval}s interval") for flap_num in range(num_flaps): # Shut down fanout interface (causes link DOWN on DUT) logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Shutting down {fanout_interface}") try: fanout.shutdown([fanout_interface]) - except: + except Exception as e: # Fallback to shell command - fanout.shell(f"configure terminal", module_ignore_errors=True) + fanout.shell("configure terminal", module_ignore_errors=True) fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) - fanout.shell(f"shutdown", module_ignore_errors=True) - fanout.shell(f"end", module_ignore_errors=True) + fanout.shell("shutdown", module_ignore_errors=True) + fanout.shell("end", module_ignore_errors=True) time.sleep(interval / 2) @@ -207,12 +207,12 @@ def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, n logger.debug(f"Flap {flap_num + 1}/{num_flaps}: Bringing up {fanout_interface}") try: fanout.no_shutdown([fanout_interface]) - except: + except Exception as e: # Fallback to shell command - fanout.shell(f"configure terminal", module_ignore_errors=True) + fanout.shell("configure terminal", module_ignore_errors=True) fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) - fanout.shell(f"no shutdown", module_ignore_errors=True) - fanout.shell(f"end", module_ignore_errors=True) + fanout.shell("no shutdown", module_ignore_errors=True) + fanout.shell("end", module_ignore_errors=True) time.sleep(interval / 2) @@ -564,7 +564,7 @@ def calculate_expected_suppression_time(suppress_threshold, reuse_threshold, dec decay_half_life_sec = decay_half_life / 1000.0 time_to_reuse = decay_half_life_sec * math.log2(ratio) logger.info(f"Expected suppression time: {time_to_reuse:.2f}s " - f"(penalty {penalty} -> {reuse_threshold}, half_life={decay_half_life}ms)") + f"(penalty {penalty} -> {reuse_threshold}, half_life={decay_half_life}ms)") return time_to_reuse else: return 0 diff --git a/tests/link_dampening/test_link_event_damping.py b/tests/link_dampening/test_link_event_damping.py index fdbdbb12cae..e6983c7a7ab 100644 --- a/tests/link_dampening/test_link_event_damping.py +++ b/tests/link_dampening/test_link_event_damping.py @@ -5,7 +5,7 @@ from tests.common.helpers.assertions import pytest_assert from tests.link_dampening.link_event_damping_utils import ( - get_dut_fronface_ports, + # get_dut_fronface_ports, configure_link_damping, verify_configuration, get_link_damping_stats, @@ -16,12 +16,12 @@ get_redis_db_entries, validate_redis_persistence, get_dampening_penalties, - verify_counter_values, - calculate_expected_suppression_time, + # verify_counter_values, + # calculate_expected_suppression_time, check_suppression_active, - inject_traffic_and_verify, + # inject_traffic_and_verify, restart_docker_container, - wait_for_condition + # wait_for_condition ) logger = logging.getLogger(__name__) @@ -78,7 +78,8 @@ def setup_teardown(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hos # Cleanup: clear damping config after each test # clear_link_damping_stats(duthost) - def test_tc01_1_normal_link_flap_event_propagation(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc01_1_normal_link_flap_event_propagation(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC01.1 - Normal Link Flap Event Propagation Verify that link up/down events propagate normally when damping is inactive. @@ -131,7 +132,7 @@ def test_tc01_2_multiple_sequential_flaps(self, duthost, duthosts, enum_rand_one time.sleep(10) - #get stats before the link flap + # get stats before the link flap stats1 = get_link_damping_stats(dut, test_intf) pre_damping_downs1 = int(stats1.get('pre_damping_down_events', 0)) pre_damping_ups1 = int(stats1.get('pre_damping_up_events', 0)) @@ -152,11 +153,12 @@ def test_tc01_2_multiple_sequential_flaps(self, duthost, duthosts, enum_rand_one logger.warning(f"Pre-damping UP events: {pre_damping_ups}") pytest_assert(pre_damping_downs >= pre_damping_downs1, - "Expected DOWN events to be recorded") + "Expected DOWN events to be recorded") pytest_assert(pre_damping_ups >= pre_damping_ups1, - "Expected UP events to be recorded") + "Expected UP events to be recorded") - def test_tc01_3_simultaneous_flaps_on_multiple_ports(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc01_3_simultaneous_flaps_on_multiple_ports(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC01.3 - Simultaneous Flaps on Multiple Ports Verify simultaneous link flaps on multiple ports are handled correctly. @@ -187,13 +189,14 @@ def test_tc01_3_simultaneous_flaps_on_multiple_ports(self, duthost, duthosts, en pre_damping_transitions = int(stats.get('pre_damping_link_transitions', 0)) logger.warning(f"Pre-damping DOWN events: {pre_damping_transitions}") pytest_assert(pre_damping_transitions > 0, - f"Expected transitions on {intf}") + f"Expected transitions on {intf}") class TestLinkEventDampingConfiguration: """Test cases for damping configuration validation (TC02-TC03)""" - def test_tc02_1_basic_link_damping_configuration(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc02_1_basic_link_damping_configuration(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC02.1 - Basic Link Damping Configuration Verify that damping configuration is applied correctly. @@ -205,11 +208,11 @@ def test_tc02_1_basic_link_damping_configuration(self, duthost, duthosts, enum_r # Configure damping with all parameters configure_link_damping(dut, test_intf, - suppress_threshold=DAMPING_CONFIG_PARAMS["suppress_threshold"], - reuse_threshold=DAMPING_CONFIG_PARAMS["reuse_threshold"], - decay_half_life=DAMPING_CONFIG_PARAMS["decay_half_life"], - max_suppress_time=DAMPING_CONFIG_PARAMS["max_suppress_time"], - flap_penalty=DAMPING_CONFIG_PARAMS["flap_penalty"]) + suppress_threshold=DAMPING_CONFIG_PARAMS["suppress_threshold"], + reuse_threshold=DAMPING_CONFIG_PARAMS["reuse_threshold"], + decay_half_life=DAMPING_CONFIG_PARAMS["decay_half_life"], + max_suppress_time=DAMPING_CONFIG_PARAMS["max_suppress_time"], + flap_penalty=DAMPING_CONFIG_PARAMS["flap_penalty"]) time.sleep(5) # Verify configuration in CONFIG_DB @@ -232,7 +235,7 @@ def test_tc02_2_config_db_persistence(self, duthost, duthosts, enum_rand_one_per time.sleep(5) # Query CONFIG_DB - config_entries = get_redis_db_entries(dut, "CONFIG_DB", f"*LINK_EVENT_DAMPING*") + config_entries = get_redis_db_entries(dut, "CONFIG_DB", "*LINK_EVENT_DAMPING*") pytest_assert(config_entries, f"No CONFIG_DB entries found for {test_intf}") logger.info(f"CONFIG_DB entries: {config_entries}") @@ -333,7 +336,8 @@ def test_tc02_6_configuration_synchronization(self, duthost, duthosts, enum_rand class TestLinkEventDampingUnsupported: """Test cases for unsupported configuration handling (TC03)""" - def test_tc03_1_decay_exceeds_max_suppress_time(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc03_1_decay_exceeds_max_suppress_time(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC03.1 - Decay Exceeds Max Suppress Time Verify damping is disabled when decay-half-life > max-suppress-time. @@ -361,8 +365,9 @@ def test_tc03_1_decay_exceeds_max_suppress_time(self, duthost, duthosts, enum_ra logger.warning(f"Post-damping propagated: {post_damping_propagated}") # All events should be propagated (damping disabled) - pytest_assert(post_damping_propagated == pre_damping_transitions or post_damping_propagated >= pre_damping_transitions - 1, - "Unsupported config should disable damping") + pytest_assert(post_damping_propagated == pre_damping_transitions or + post_damping_propagated >= pre_damping_transitions - 1, + "Unsupported config should disable damping") def test_tc03_2_zero_flap_penalty(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC03.2 - Zero Flap Penalty @@ -487,7 +492,8 @@ def test_tc04_1_basic_mixed_configuration(self, duthost, duthosts, enum_rand_one logger.info("Mixed configuration applied successfully") - def test_tc04_2_simultaneous_flaps_damped_vs_undamped(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc04_2_simultaneous_flaps_damped_vs_undamped(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC04.2 - Simultaneous Flaps (Damped vs Undamped) Compare behavior of damped and undamped ports. @@ -522,7 +528,7 @@ def test_tc04_2_simultaneous_flaps_damped_vs_undamped(self, duthost, duthosts, e # Undamped should propagate more events pytest_assert(undamped_propagated >= damped_propagated, - "Undamped interface should propagate more events") + "Undamped interface should propagate more events") def test_tc04_3_different_damping_profiles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC04.3 - Different Damping Profiles @@ -603,7 +609,8 @@ def test_tc04_6_large_scale_mixed_configuration(self, duthost, duthosts, enum_ra class TestLinkEventDampingOperationalState: """Test cases for operational state accuracy (TC05)""" - def test_tc05_1_operational_state_frozen_during_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc05_1_operational_state_frozen_during_suppression(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC05.1 - Operational State Frozen During Suppression Verify operational state remains frozen while damping is active. @@ -624,9 +631,10 @@ def test_tc05_1_operational_state_frozen_during_suppression(self, duthost, dutho logger.info(f"During suppression - Op state: {op_state_during}, Phys state: {phys_state}") pytest_assert(op_state_during != phys_state or op_state_during == "down", - "Operational state should be frozen during suppression") + "Operational state should be frozen during suppression") - def test_tc05_2_operational_state_updates_after_suppression_ends(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc05_2_operational_state_updates_after_suppression_ends(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC05.2 - Operational State Updates After Suppression Ends Verify operational state updates after suppression ends. @@ -652,9 +660,10 @@ def test_tc05_2_operational_state_updates_after_suppression_ends(self, duthost, logger.info(f"After suppression - Op state: {op_state}, Phys state: {phys_state}") pytest_assert(op_state == phys_state, - "Operational state should match physical state after suppression ends") + "Operational state should match physical state after suppression ends") - def test_tc05_3_physical_vs_operational_state_divergence(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc05_3_physical_vs_operational_state_divergence(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC05.3 - Physical vs Operational State Divergence During Suppression Verify physical and operational states diverge during suppression. @@ -681,7 +690,8 @@ def test_tc05_3_physical_vs_operational_state_divergence(self, duthost, duthosts logger.info(f"Physical state: {phys_state}, Operational state: {op_state}") # States should diverge if suppression is active - def test_tc05_4_penalty_decay_and_state_recovery(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc05_4_penalty_decay_and_state_recovery(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC05.4 - Penalty Decay and State Recovery Verify state recovers as penalty decays. @@ -706,7 +716,7 @@ def test_tc05_4_penalty_decay_and_state_recovery(self, duthost, duthosts, enum_r # Penalty should decay pytest_assert(current_penalty < initial_penalty or current_penalty == 0, - "Penalty should decay over time") + "Penalty should decay over time") def test_tc05_5_multiple_suppression_cycles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC05.5 - Multiple Suppression Cycles @@ -729,10 +739,12 @@ def test_tc05_5_multiple_suppression_cycles(self, duthost, duthosts, enum_rand_o logger.info("Multiple suppression cycles completed") + class TestLinkEventDampingFrequency: """Test cases for flap frequency effects (TC06)""" - def test_tc06_1_frequent_flaps_longer_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc06_1_frequent_flaps_longer_suppression(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC06.1 - Frequent Flaps Longer Suppression Verify frequent flaps result in longer suppression. @@ -759,7 +771,8 @@ def test_tc06_1_frequent_flaps_longer_suppression(self, duthost, duthosts, enum_ # Should be longer than minimal pytest_assert(suppression_duration > 5, "Suppression should last a reasonable time") - def test_tc06_2_infrequent_flaps_shorter_suppression(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc06_2_infrequent_flaps_shorter_suppression(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC06.2 - Infrequent Flaps Shorter Suppression Verify infrequent flaps result in shorter suppression. @@ -818,7 +831,8 @@ def test_tc06_4_decay_rate_same_for_both(self, duthost, duthosts, enum_rand_one_ logger.info(f"Penalty at t=0: {penalty_t0}, t=5: {penalty_t5}, t=10: {penalty_t10}") - def test_tc06_5_recovery_time_proportional_to_frequency(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc06_5_recovery_time_proportional_to_frequency(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC06.5 - Recovery Time Proportional to Frequency Verify recovery time varies with flap frequency. @@ -852,7 +866,8 @@ def test_tc06_6_mixed_pattern_suppression(self, duthost, duthosts, enum_rand_one logger.info("Mixed pattern suppression tested") - def test_tc06_7_threshold_crossing_different_timing(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc06_7_threshold_crossing_different_timing(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC06.7 - Threshold Crossing Different Timing Verify different timing for threshold crossing. @@ -875,7 +890,8 @@ def test_tc06_7_threshold_crossing_different_timing(self, duthost, duthosts, enu class TestLinkEventDampingCounters: """Test cases for counter verification (TC07)""" - def test_tc07_1_pre_damping_link_transitions_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_1_pre_damping_link_transitions_counter(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.1 - Pre-Damping Link Transitions Counter Verify pre-damping link transitions counter. @@ -899,7 +915,8 @@ def test_tc07_1_pre_damping_link_transitions_counter(self, duthost, duthosts, en logger.info(f"Pre-damping link transitions: {transitions}") pytest_assert(transitions > 0, "Pre-damping transitions should be recorded") - def test_tc07_2_post_damping_propagated_transitions_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_2_post_damping_propagated_transitions_counter(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.2 - Post-Damping Propagated Transitions Counter Verify post-damping propagated transitions counter. @@ -967,7 +984,8 @@ def test_tc07_4_pre_damping_down_events_counter(self, duthost, duthosts, enum_ra logger.info(f"Pre-damping DOWN events: {down_events}") pytest_assert(down_events > 0, "DOWN events should be recorded") - def test_tc07_5_post_damping_up_advertised_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_5_post_damping_up_advertised_counter(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.5 - Post-Damping UP Advertised Counter Verify post-damping UP advertised counter. @@ -989,7 +1007,8 @@ def test_tc07_5_post_damping_up_advertised_counter(self, duthost, duthosts, enum logger.info(f"Post-damping UP advertised: {up_advertised}") - def test_tc07_6_post_damping_down_advertised_counter(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_6_post_damping_down_advertised_counter(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.6 - Post-Damping DOWN Advertised Counter Verify post-damping DOWN advertised counter. @@ -1011,7 +1030,8 @@ def test_tc07_6_post_damping_down_advertised_counter(self, duthost, duthosts, en logger.info(f"Post-damping DOWN advertised: {down_advertised}") - def test_tc07_7_counter_consistency_across_cycles(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_7_counter_consistency_across_cycles(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.7 - Counter Consistency Across Cycles Verify counters remain consistent across multiple cycles. @@ -1037,7 +1057,8 @@ def test_tc07_7_counter_consistency_across_cycles(self, duthost, duthosts, enum_ logger.info(f"Cycle 1 transitions: {count1}, Cycle 2 transitions: {count2}") pytest_assert(count2 >= count1, "Counter should monotonically increase") - def test_tc07_8_counter_increments_proportional_to_events(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_8_counter_increments_proportional_to_events(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.8 - Counter Increments Proportional to Events Verify counter increments are proportional to events. @@ -1059,7 +1080,8 @@ def test_tc07_8_counter_increments_proportional_to_events(self, duthost, duthost logger.info(f"Transitions for 10 flaps: {transitions}") pytest_assert(transitions >= 10, "Counter should reflect number of events") - def test_tc07_9_suppressed_events_not_in_post_damping(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc07_9_suppressed_events_not_in_post_damping(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC07.9 - Suppressed Events Not in Post-Damping Verify suppressed events are not counted in post-damping. @@ -1081,7 +1103,7 @@ def test_tc07_9_suppressed_events_not_in_post_damping(self, duthost, duthosts, e logger.info(f"Pre-damping: {pre_transitions}, Post-damping: {post_transitions}") pytest_assert(post_transitions <= pre_transitions, - "Post-damping should not include suppressed events") + "Post-damping should not include suppressed events") def test_tc07_10_counter_reset_and_recovery(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC07.10 - Counter Reset and Recovery @@ -1110,7 +1132,8 @@ def test_tc07_10_counter_reset_and_recovery(self, duthost, duthosts, enum_rand_o class TestLinkEventDampingTimeline: """Test cases for timeline validation (TC09)""" - def test_tc09_1_timeline_event_sequence_execution(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc09_1_timeline_event_sequence_execution(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC09.1 - Timeline Event Sequence Execution Execute deterministic timeline of events. @@ -1150,11 +1173,11 @@ def test_tc09_1_timeline_event_sequence_execution(self, duthost, duthosts, enum_ }) - class TestLinkEventDampingPersistence: """Test cases for persistence across reboots and docker restarts (TC10)""" - def test_tc10_1_damping_config_persists_after_reboot(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc10_1_damping_config_persists_after_reboot(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC10.1 - Damping Config Persists After Reboot Verify damping configuration persists after device reboot. @@ -1181,7 +1204,8 @@ def test_tc10_1_damping_config_persists_after_reboot(self, duthost, duthosts, en logger.info("Configuration persisted after reboot") - def test_tc10_2_damping_functionality_after_reboot(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc10_2_damping_functionality_after_reboot(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC10.2 - Damping Functionality After Reboot Verify damping functionality works correctly after reboot. @@ -1266,8 +1290,8 @@ def test_tc10_4_multiple_reboot_cycles(self, duthost, duthosts, enum_rand_one_pe logger.info("Multiple reboot cycles completed successfully") - - def test_tc10_5_concurrent_damping_multiple_ports(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): + def test_tc10_5_concurrent_damping_multiple_ports(self, duthost, duthosts, + enum_rand_one_per_hwsku_frontend_hostname): """TC10.5 - Concurrent Damping Multiple Ports Verify concurrent damping on multiple ports survives reboot. From 8e06062ca75b7722b5eed92cd8638bae9de2497d Mon Sep 17 00:00:00 2001 From: Muneer CH Date: Thu, 18 Jun 2026 06:01:18 +0000 Subject: [PATCH 4/5] Addressing review comments Signed-off-by: Muneer CH --- tests/link_dampening/conftest.py | 5 +---- tests/link_dampening/link_event_damping_utils.py | 2 ++ tests/link_dampening/test_link_event_damping.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/link_dampening/conftest.py b/tests/link_dampening/conftest.py index 61e0a7b29a4..2a929b0616a 100644 --- a/tests/link_dampening/conftest.py +++ b/tests/link_dampening/conftest.py @@ -6,9 +6,6 @@ import pytest # from tests.common.helpers.assertions import pytest_assert as pt_assert -# from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports - -from tests.common.helpers.assertions import pytest_assert from tests.link_dampening.link_event_damping_utils import get_dut_fronface_ports logger = logging.getLogger(__name__) @@ -62,7 +59,7 @@ def cleanup_link_damping(duthosts, enum_rand_one_per_hwsku_frontend_hostname): dut = duthosts[enum_rand_one_per_hwsku_frontend_hostname] # Clear link damping stats dut.shell("redis-cli -n 0 --scan --match 'LINK_DAMPING_STATS:*' | xargs redis-cli -n 0 DEL", - module_ignore_errors=True) + module_ignore_errors=True) logger.info("Cleaned up link damping statistics") except Exception as e: logger.warning(f"Could not cleanup link damping stats: {e}") diff --git a/tests/link_dampening/link_event_damping_utils.py b/tests/link_dampening/link_event_damping_utils.py index a6d0e56de17..80a44894f22 100644 --- a/tests/link_dampening/link_event_damping_utils.py +++ b/tests/link_dampening/link_event_damping_utils.py @@ -195,6 +195,7 @@ def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, n try: fanout.shutdown([fanout_interface]) except Exception as e: + logger.warning(f"Native shutdown failed, falling back to shell. Error: {e}") # Fallback to shell command fanout.shell("configure terminal", module_ignore_errors=True) fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) @@ -208,6 +209,7 @@ def generate_link_flap(dut, dut_interface, fanout=None, fanout_interface=None, n try: fanout.no_shutdown([fanout_interface]) except Exception as e: + logger.warning(f"Native shutdown failed, falling back to shell. Error: {e}") # Fallback to shell command fanout.shell("configure terminal", module_ignore_errors=True) fanout.shell(f"interface {fanout_interface}", module_ignore_errors=True) diff --git a/tests/link_dampening/test_link_event_damping.py b/tests/link_dampening/test_link_event_damping.py index e6983c7a7ab..b333846c8f4 100644 --- a/tests/link_dampening/test_link_event_damping.py +++ b/tests/link_dampening/test_link_event_damping.py @@ -115,7 +115,7 @@ def test_tc01_1_normal_link_flap_event_propagation(self, duthost, duthosts, op_state = get_interface_operational_state(dut, test_intf) phys_state = get_interface_physical_state(dut, test_intf) pytest_assert(op_state == phys_state, - f"Operational state {op_state} should match physical state {phys_state}") + f"Operational state {op_state} should match physical state {phys_state}") def test_tc01_2_multiple_sequential_flaps(self, duthost, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """TC01.2 - Multiple Sequential Flaps @@ -333,6 +333,7 @@ def test_tc02_6_configuration_synchronization(self, duthost, duthosts, enum_rand logger.info("Configuration synchronisation with CONFIG DB verified") + class TestLinkEventDampingUnsupported: """Test cases for unsupported configuration handling (TC03)""" From ee4d5ce462c6e90451544f4fa2ee040b5858ee51 Mon Sep 17 00:00:00 2001 From: Muneer CH Date: Thu, 18 Jun 2026 06:22:56 +0000 Subject: [PATCH 5/5] Removing trailing space errors Signed-off-by: Muneer CH --- tests/link_dampening/link_event_damping_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/link_dampening/link_event_damping_utils.py b/tests/link_dampening/link_event_damping_utils.py index 80a44894f22..8d17dc24807 100644 --- a/tests/link_dampening/link_event_damping_utils.py +++ b/tests/link_dampening/link_event_damping_utils.py @@ -455,7 +455,7 @@ def get_dampening_penalties(dut, interface): oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) oid = oid_result['stdout'].strip() - + cmd = f"redis-cli -n 6 HGET 'LINK_EVENT_DAMPING_STATS|{oid}' 'current_penalty'" result = dut.shell(cmd, module_ignore_errors=True) @@ -489,7 +489,7 @@ def check_suppression_active(dut, interface): oid_result = dut.shell(get_oid_cmd, module_ignore_errors=True) oid = oid_result['stdout'].strip() - + cmd = f"redis-cli -n 6 HGET 'LINK_EVENT_DAMPING_STATS|{oid}' 'is_damping_active'" result = dut.shell(cmd, module_ignore_errors=True)