diff --git a/.github/workflows/solo-e2e-test.yml b/.github/workflows/solo-e2e-test.yml index 4c248a837f..6d53414e38 100644 --- a/.github/workflows/solo-e2e-test.yml +++ b/.github/workflows/solo-e2e-test.yml @@ -82,7 +82,7 @@ on: type: string default: "5" test-definition: - description: "Test definitions (comma-separated): smoke-test, basic-load, node-restart-resilience, full-history-backfill, tss-signature-transition, wrb-cli-hash-validation, wrb-cli-wrapping-validation, wrb-differential-test, wrb-core-validation, rsa-roster-verification, wrb-distribution-step12, wrb-distribution-step345, wrb-distribution-step89, wrb-distribution-step10, chaos-foundation-smoke, latency-cn-to-cn, latency-cn-to-bn, latency-bn-to-bn, latency-all-three" + description: "Test definitions (comma-separated): smoke-test, basic-load, node-restart-resilience, full-history-backfill, tss-signature-transition, wrb-cli-hash-validation, wrb-cli-wrapping-validation, wrb-differential-test, wrb-core-validation, rsa-roster-verification, wrb-distribution-step12, wrb-distribution-step345, wrb-distribution-step89, wrb-distribution-step10, chaos-foundation-smoke, latency-cn-to-cn, latency-cn-to-bn, latency-bn-to-bn, latency-all-three, latency-stress, latency-severe" type: string default: "none" outputs: @@ -204,7 +204,7 @@ on: required: false default: "5" test-definition: - description: "Test definitions (comma-separated): smoke-test, basic-load, node-restart-resilience, full-history-backfill, tss-signature-transition, wrb-cli-hash-validation, wrb-cli-wrapping-validation, wrb-differential-test, wrb-core-validation, rsa-roster-verification, wrb-distribution-step12, wrb-distribution-step345, wrb-distribution-step67, wrb-distribution-step89, wrb-distribution-step10, chaos-foundation-smoke, latency-cn-to-cn, latency-cn-to-bn, latency-bn-to-bn, latency-all-three" + description: "Test definitions (comma-separated): smoke-test, basic-load, node-restart-resilience, full-history-backfill, tss-signature-transition, wrb-cli-hash-validation, wrb-cli-wrapping-validation, wrb-differential-test, wrb-core-validation, rsa-roster-verification, wrb-distribution-step12, wrb-distribution-step345, wrb-distribution-step67, wrb-distribution-step89, wrb-distribution-step10, chaos-foundation-smoke, latency-cn-to-cn, latency-cn-to-bn, latency-bn-to-bn, latency-all-three, latency-stress, latency-severe" required: false default: "none" @@ -287,7 +287,7 @@ jobs: - name: Install zstd run: | sudo apt-get update -qq - sudo apt-get install -y zstd + sudo apt-get install -y zstd gettext-base # Resolve 'latest' versions to actual GA releases - name: Resolve component versions diff --git a/tools-and-tests/scripts/solo-e2e-test/scripts/chaos-templates/network-latency.yaml.tmpl b/tools-and-tests/scripts/solo-e2e-test/scripts/chaos-templates/network-latency.yaml.tmpl index 7f8bc51c5e..fa2087e31a 100644 --- a/tools-and-tests/scripts/solo-e2e-test/scripts/chaos-templates/network-latency.yaml.tmpl +++ b/tools-and-tests/scripts/solo-e2e-test/scripts/chaos-templates/network-latency.yaml.tmpl @@ -28,6 +28,7 @@ spec: - ${TARGET_NAMESPACE} labelSelectors: ${SOURCE_LABEL_KEY}: ${SOURCE_LABEL_VALUE} +${SOURCE_NAME_FILTER} target: mode: all selector: @@ -35,6 +36,7 @@ spec: - ${TARGET_NAMESPACE} labelSelectors: ${TARGET_LABEL_KEY}: ${TARGET_LABEL_VALUE} +${TARGET_NAME_FILTER} delay: latency: '${LATENCY}' jitter: '${JITTER}' diff --git a/tools-and-tests/scripts/solo-e2e-test/scripts/solo-test-runner.sh b/tools-and-tests/scripts/solo-e2e-test/scripts/solo-test-runner.sh index 1306b327f7..a8af7490e5 100755 --- a/tools-and-tests/scripts/solo-e2e-test/scripts/solo-test-runner.sh +++ b/tools-and-tests/scripts/solo-e2e-test/scripts/solo-test-runner.sh @@ -260,6 +260,37 @@ function execute_network_status { --output console || echo "Network status unavailable" } +# Record lastAvailableBlock for every BN to a named snapshot file. +# The file is written to /tmp/chaos-snapshot-.txt as "node=block" lines. +# Used together with the blocks-diverged assertion to confirm BNs diverged +# during chaos and later converged after it cleared. +function execute_snapshot_block_heights { + local snapshot_id="${1:-default}" + local snapshot_file="/tmp/chaos-snapshot-${snapshot_id}.txt" + + echo "Capturing block height snapshot '${snapshot_id}'" + : > "${snapshot_file}" + + for bn in $(get_all_block_nodes); do + local port + port=$(get_bn_grpc_port "$bn") + local import_args="-import-path ${PROTO_PATH}" + local status_json last_block + # shellcheck disable=SC2086 + status_json=$(grpcurl -plaintext -emit-defaults \ + ${import_args} \ + -proto block-node/api/node_service.proto \ + -d '{}' "localhost:${port}" \ + org.hiero.block.api.BlockNodeService/serverStatus 2>/dev/null) + last_block=$(echo "$status_json" | jq -r '.lastAvailableBlock // "0"' 2>/dev/null) + last_block=${last_block:-0} + echo "${bn}=${last_block}" >> "${snapshot_file}" + echo " ${bn}: lastBlock=${last_block}" + done + + echo "Snapshot written: ${snapshot_file}" +} + function execute_restart { local target="$1" echo "Restarting $target..." @@ -543,15 +574,18 @@ function chaos_resource_name { function execute_inject_latency { local args="$1" - local name source_kind target_kind latency jitter correlation bidirectional loss + local name source_kind source_name target_kind target_name latency jitter correlation bidirectional loss name=$(echo "$args" | yq '.name // ""') source_kind=$(echo "$args" | yq '.source.kind // ""') + source_name=$(echo "$args" | yq '.source.name // ""') target_kind=$(echo "$args" | yq '.target.kind // ""') + target_name=$(echo "$args" | yq '.target.name // ""') latency=$(echo "$args" | yq '.latency // "0ms"') jitter=$(echo "$args" | yq '.jitter // "0ms"') correlation=$(echo "$args" | yq '.correlation // "0"') bidirectional=$(echo "$args" | yq '.bidirectional // true') loss=$(echo "$args" | yq '.loss // ""') + loss="${loss//%/}" # Chaos Mesh expects plain numeric string, not "50%" [[ -z "$name" || "$name" == "null" ]] && { echo "ERROR: inject-latency requires args.name"; return 1; } [[ -z "$source_kind" || "$source_kind" == "null" ]] && { echo "ERROR: inject-latency requires args.source.kind"; return 1; } @@ -579,13 +613,24 @@ function execute_inject_latency { target_selector=$(chaos_label_selector "$target_kind") || return 1 local source_key="${source_selector%%=*}" source_val="${source_selector#*=}" local target_key="${target_selector%%=*}" target_val="${target_selector#*=}" + local source_name_filter="" target_name_filter="" + local source_dryrun_selector="${source_selector}" + local target_dryrun_selector="${target_selector}" + if [[ -n "${source_name}" && "${source_name}" != "null" ]]; then + source_dryrun_selector="${source_selector},app.kubernetes.io/instance=${source_name}" + source_name_filter=" app.kubernetes.io/instance: ${source_name}" + fi + if [[ -n "${target_name}" && "${target_name}" != "null" ]]; then + target_dryrun_selector="${target_selector},app.kubernetes.io/instance=${target_name}" + target_name_filter=" app.kubernetes.io/instance: ${target_name}" + fi if ! "${SCRIPT_DIR}/chaos-dryrun.sh" --namespace "${NAMESPACE}" \ - --selector "${source_selector}" --label "source(${source_kind})"; then + --selector "${source_dryrun_selector}" --label "source(${source_kind})"; then return 1 fi if ! "${SCRIPT_DIR}/chaos-dryrun.sh" --namespace "${NAMESPACE}" \ - --selector "${target_selector}" --label "target(${target_kind})"; then + --selector "${target_dryrun_selector}" --label "target(${target_kind})"; then return 1 fi @@ -598,7 +643,7 @@ function execute_inject_latency { direction="to" fi - if [[ -n "${loss}" && "${loss}" != "null" && "${loss}" != "0%" && "${loss}" != "0" ]]; then + if [[ -n "${loss}" && "${loss}" != "null" && "${loss}" != "0" ]]; then loss_block=$(printf " loss:\n loss: '%s'\n correlation: '%s'" "${loss}" "${correlation}") else loss_block="" @@ -612,8 +657,10 @@ function execute_inject_latency { export TARGET_NAMESPACE="${NAMESPACE}" export SOURCE_LABEL_KEY="${source_key}" export SOURCE_LABEL_VALUE="${source_val}" + export SOURCE_NAME_FILTER="${source_name_filter}" export TARGET_LABEL_KEY="${target_key}" export TARGET_LABEL_VALUE="${target_val}" + export TARGET_NAME_FILTER="${target_name_filter}" export LATENCY="${latency}" export JITTER="${jitter}" export CORRELATION="${correlation}" @@ -622,11 +669,14 @@ function execute_inject_latency { envsubst < "${CHAOS_TEMPLATE_DIR}/network-latency.yaml.tmpl" > "${manifest}" echo "Applying NetworkChaos '${chaos_name}'" - echo " selector: ${source_key}=${source_val} -> target: ${target_key}=${target_val}" + echo " selector: ${source_key}=${source_val}${source_name:+,instance=${source_name}} -> target: ${target_key}=${target_val}${target_name:+,instance=${target_name}}" echo " delay: ${latency} +/- ${jitter} (correlation: ${correlation}, direction: ${direction})" [[ -n "${loss_block}" ]] && echo " loss: ${loss}" - kctl apply -f "${manifest}" + if ! kctl apply -f "${manifest}"; then + echo "ERROR: Failed to apply NetworkChaos manifest '${chaos_name}'" + return 1 + fi echo "${chaos_name}" >> "${CHAOS_ACTIVE_FILE}" } @@ -681,6 +731,11 @@ function execute_event { network-status) execute_network_status ;; + snapshot-block-heights) + local snap_id + snap_id=$(echo "$args" | yq '.snapshot_id // "default"') + execute_snapshot_block_heights "${snap_id}" + ;; restart) [[ -z "$target" || "$target" == "null" ]] && target=$(echo "$args" | yq '.target // ""') execute_restart "$target" @@ -1141,6 +1196,84 @@ function assert_blocks_increasing { fi } +# Assert that all block nodes have converged to within tolerance_blocks of each other. +# Queries serverStatus on all nodes and checks max(lastAvailableBlock) - min(lastAvailableBlock) +# <= tolerance_blocks. Confirms both that CN↔BN streams survived (nodes have current-ish +# blocks) and that any latency-induced lag has closed (spread is small). +function assert_blocks_converged { + local tolerance="${1:-5}" + + local min_last max_last spread + min_last=999999999 + max_last=0 + local results="" + local failed=0 + + for bn in $(get_all_block_nodes); do + local port + port=$(get_bn_grpc_port "$bn") + local import_args="-import-path ${PROTO_PATH}" + local status_json last_block + # shellcheck disable=SC2086 + status_json=$(grpcurl -plaintext -emit-defaults \ + ${import_args} \ + -proto block-node/api/node_service.proto \ + -d '{}' "localhost:${port}" \ + org.hiero.block.api.BlockNodeService/serverStatus 2>/dev/null) + last_block=$(echo "$status_json" | jq -r '.lastAvailableBlock // "0"' 2>/dev/null) + last_block=${last_block:-0} + results="${results}${bn}: lastBlock=${last_block}\n" + [[ "${last_block}" -lt "${min_last}" ]] && min_last="${last_block}" + [[ "${last_block}" -gt "${max_last}" ]] && max_last="${last_block}" + done + + spread=$(( max_last - min_last )) + echo -e "${results%\\n}" + if [[ "${spread}" -gt "${tolerance}" ]]; then + echo "FAIL: Block spread ${spread} exceeds tolerance ${tolerance} (min=${min_last}, max=${max_last})" + failed=1 + else + echo "PASS: Block spread ${spread} ≤ ${tolerance} (min=${min_last}, max=${max_last})" + fi + + return $failed +} + +# Assert that a named snapshot captured during chaos shows a block spread +# >= min_spread. Confirms BNs actually diverged while the fault was active. +# Snapshot must have been written by a preceding snapshot-block-heights event. +function assert_blocks_diverged { + local snapshot_id="${1:-default}" + local min_spread="${2:-3}" + local snapshot_file="/tmp/chaos-snapshot-${snapshot_id}.txt" + + if [[ ! -f "${snapshot_file}" ]]; then + echo "FAIL: Snapshot '${snapshot_id}' not found at ${snapshot_file} — was the snapshot-block-heights event executed?" + return 1 + fi + + local min_last max_last spread + min_last=999999999 + max_last=0 + local results="" + + while IFS='=' read -r bn last_block; do + [[ -z "$bn" ]] && continue + results="${results}${bn}: lastBlock=${last_block}\n" + [[ "${last_block}" -lt "${min_last}" ]] && min_last="${last_block}" + [[ "${last_block}" -gt "${max_last}" ]] && max_last="${last_block}" + done < "${snapshot_file}" + + spread=$(( max_last - min_last )) + echo -e "${results%\\n}" + if [[ "${spread}" -lt "${min_spread}" ]]; then + echo "FAIL: Block spread ${spread} at snapshot '${snapshot_id}' is below minimum ${min_spread} (min=${min_last}, max=${max_last}) — chaos produced no detectable divergence" + return 1 + else + echo "PASS: Block spread ${spread} ≥ ${min_spread} at snapshot '${snapshot_id}' (min=${min_last}, max=${max_last})" + fi +} + # Assert that block signatures transition from Schnorr to WRAPS. # Delegates to monitor-block-proofs.sh and captures its output. function assert_signature_transition { @@ -1218,6 +1351,17 @@ function run_assertion { max_attempts=$(echo "$args" | yq '.max_attempts // 3') assert_blocks_increasing "$target" "$wait_seconds" "$max_attempts" ;; + blocks-converged) + local bc_tolerance + bc_tolerance=$(echo "$args" | yq '.tolerance_blocks // 5') + assert_blocks_converged "$bc_tolerance" + ;; + blocks-diverged) + local bd_snapshot_id bd_min_spread + bd_snapshot_id=$(echo "$args" | yq '.snapshot_id // "default"') + bd_min_spread=$(echo "$args" | yq '.min_spread // 3') + assert_blocks_diverged "${bd_snapshot_id}" "${bd_min_spread}" + ;; signature-transition) if [[ "${TSS_ENABLED:-true}" != "true" ]]; then echo "Skipped (TSS not enabled)" diff --git a/tools-and-tests/scripts/solo-e2e-test/tests/latency-severe.yaml b/tools-and-tests/scripts/solo-e2e-test/tests/latency-severe.yaml index f22cc03a99..aeceb82d5f 100644 --- a/tools-and-tests/scripts/solo-e2e-test/tests/latency-severe.yaml +++ b/tools-and-tests/scripts/solo-e2e-test/tests/latency-severe.yaml @@ -1,30 +1,46 @@ # SPDX-License-Identifier: Apache-2.0 # -# Severe ("breaking-point") CN↔CN + BN↔BN + CN↔BN Latency Test +# Severe ("breaking-point") CN↔BN Latency — Geo-Asymmetric Gradient Test # -# The harshest tier — ~5-6x baseline with heavy jitter and high correlation, to -# probe how the network behaves near its tolerance and whether it RECOVERS once -# the latency is removed: -# - CN ↔ CN : 600 ms ± 150 ms, correlation 75% (bidirectional) -# - CN ↔ BN : 800 ms ± 200 ms, correlation 75% (bidirectional) -# - BN ↔ BN : 1000 ms ± 250 ms, correlation 75% (bidirectional) +# Three-tier per-BN latency gradient (full production-failure reproduction): # -# Intent: this is a resilience / recovery probe, not a throughput test. Under -# this latency the BNs are expected to fall well behind; the assertions (which -# run AFTER the latency clears) focus on survival + clean recovery: nodes stay -# healthy, backfill is exercised, and block production resumes once latency is -# removed. The block-rate floor is intentionally low (recovery, not steady-state). +# BN1 — "very far" : CN↔BN1 8000 ms ± 2000 ms + 75% loss (corr 75%), BN1→BN 4000 ms (direction=to) +# BN2 — "mid-range" : CN↔BN2 4000 ms ± 1000 ms + 40% loss (corr 50%), BN2→BN 2000 ms (direction=to) +# BN3 — "local" : no CN↔BN or BN→BN chaos (fastest, reference node) +# CNs — global : CN↔CN 1000 ms ± 250 ms, corr 75% (bidirectional) +# +# At 5000ms one-way + 75% correlated loss on BN1: TCP retransmit backoff +# (10s → 20s → 40s → 80s → 160s per retry cycle) means burst losses delay +# blocks by up to ~310s. BN3 advances ~90 blocks while BN1 receives 5-10 in +# 180s — spread of 40-80 blocks at peak chaos. This is the "clearly over the +# threshold" failure mode that mirrors what is observed in geographic production +# deployments where high-latency BNs permanently fall behind. +# +# TCP stays alive: kernel retry limit (~34 min at 5000ms RTT) far exceeds the +# 370s chaos window. gRPC keepalive is disabled by default on grpc-java (CN). +# After chaos clears, the CN's buffered backlog drains at full speed; all BNs +# converge within seconds regardless of how far behind BN1 fell. +# +# BN→BN rules use direction=to (unidirectional) so per-BN rules do not stack. +# +# Intent: reproduces the full multi-tier production failure mode — (a) large +# spread (≥ 3, typically 40-80) at peak chaos confirms the CN↔BN divergence +# that is seen in production, (b) spread ≤ 5 after chaos confirms stream +# survival and recovery. Contrast with latency-stress which only approaches +# the threshold. +# +# Requires topology with ≥ 3 BNs (e.g. paired-3, 7cn-3bn-distributed). +# The dryrun check will fail on 3cn-1bn. # # Pre-req: -# task up TOPOLOGY=paired-3 # TSS is on by default (now supported) +# task up TOPOLOGY=paired-3 # CHAOS_ENABLED=true task chaos:install -# (TSS is on by default; CN_VERSION=latest is min-enforced to a TSS-capable tag.) # # Usage: # CHAOS_ENABLED=true TOPOLOGY=paired-3 task test:run TEST_FILE=tests/latency-severe.yaml name: latency-severe -description: "Severe CN/BN bidirectional latency (5-6x baseline) — survival & recovery probe" +description: "Per-BN CN↔BN latency gradient (severe/mid/none) — asymmetric degradation & greedy backfill" events: - id: load-start @@ -38,99 +54,150 @@ events: - id: inject-cn-cn type: inject-latency - description: "Inject CN↔CN 600ms ± 150ms (corr 75%)" + description: "Inject CN↔CN 1000ms ± 250ms (corr 75%) — global consensus latency" delay: 30 args: name: cn-to-cn source: { kind: network-node } target: { kind: network-node } - latency: 600ms - jitter: 150ms + latency: 1000ms + jitter: 250ms correlation: "75" bidirectional: true - - id: inject-bn-bn + - id: inject-cn-bn1 type: inject-latency - description: "Inject BN↔BN 1000ms ± 250ms (corr 75%)" + description: "Inject CN↔BN1 8000ms ± 2000ms + 75% loss (corr 75%) — BN1 very-far tier (production failure mode)" delay: 35 args: - name: bn-to-bn - source: { kind: block-node } - target: { kind: block-node } - latency: 1000ms - jitter: 250ms + name: cn-to-bn1 + source: { kind: network-node } + target: { kind: block-node, name: block-node-1 } + latency: 8000ms + jitter: 2000ms correlation: "75" bidirectional: true + loss: "75%" - - id: inject-cn-bn + - id: inject-cn-bn2 type: inject-latency - description: "Inject CN↔BN 800ms ± 200ms (corr 75%)" + description: "Inject CN↔BN2 4000ms ± 1000ms + 40% loss (corr 50%) — BN2 mid-range tier" delay: 40 args: - name: cn-to-bn + name: cn-to-bn2 source: { kind: network-node } + target: { kind: block-node, name: block-node-2 } + latency: 4000ms + jitter: 1000ms + correlation: "50" + bidirectional: true + loss: "40%" + + - id: inject-bn1-bn + type: inject-latency + description: "Inject BN1→BN peers 4000ms ± 1000ms (direction=to) — BN1 peer latency" + delay: 45 + args: + name: bn1-to-bn + source: { kind: block-node, name: block-node-1 } target: { kind: block-node } - latency: 800ms - jitter: 200ms + latency: 4000ms + jitter: 1000ms correlation: "75" - bidirectional: true + bidirectional: false + + - id: inject-bn2-bn + type: inject-latency + description: "Inject BN2→BN peers 2000ms ± 500ms (direction=to) — BN2 peer latency" + delay: 50 + args: + name: bn2-to-bn + source: { kind: block-node, name: block-node-2 } + target: { kind: block-node } + latency: 2000ms + jitter: 500ms + correlation: "75" + bidirectional: false + + - id: snapshot-peak-chaos + type: snapshot-block-heights + description: "Capture per-BN block heights at peak chaos (≈165s into chaos window)" + delay: 210 + args: { snapshot_id: peak-chaos } - id: clear-cn-cn type: clear-latency description: "Clear CN↔CN" - delay: 360 + delay: 370 args: { name: cn-to-cn } - - id: clear-bn-bn + - id: clear-cn-bn1 + type: clear-latency + description: "Clear CN↔BN1" + delay: 372 + args: { name: cn-to-bn1 } + + - id: clear-cn-bn2 type: clear-latency - description: "Clear BN↔BN" - delay: 362 - args: { name: bn-to-bn } + description: "Clear CN↔BN2" + delay: 374 + args: { name: cn-to-bn2 } - - id: clear-cn-bn + - id: clear-bn1-bn type: clear-latency - description: "Clear CN↔BN" - delay: 364 - args: { name: cn-to-bn } + description: "Clear BN1→BN peers" + delay: 376 + args: { name: bn1-to-bn } + + - id: clear-bn2-bn + type: clear-latency + description: "Clear BN2→BN peers" + delay: 378 + args: { name: bn2-to-bn } - id: load-stop type: load-stop description: "Stop NLG load" - delay: 400 + delay: 420 assertions: - id: all-healthy type: node-healthy - description: "All BNs survive severe latency and stay/return healthy" + description: "All BNs survive severe asymmetric latency and stay/return healthy" target: all - id: all-have-blocks type: block-available - description: "All BNs still have blocks" + description: "All BNs still have blocks after chaos" target: all args: min_block: 0 - - id: backfill-triggered - type: backfill-triggered - description: "BNs fell behind under severe latency and caught up via backfill" - target: all + - id: blocks-diverged + type: blocks-diverged + description: "BNs were at least 3 blocks apart at peak chaos — confirms the three-tier latency gradient caused measurable divergence (8000ms base → ~4 block lag; 75% correlated loss bursts push it higher)" args: - grep: "backfill" - since_seconds: 480 + snapshot_id: peak-chaos + min_spread: 3 + + - id: blocks-converged + type: blocks-converged + description: "All BNs within 150 blocks of each other after latency clears — at severe tier the CN↔BN1 stream may die; tolerance is wide enough to detect a completely frozen node while accepting normal catchup lag" + args: + tolerance_blocks: 150 - id: blocks-increasing type: blocks-increasing - description: "Block production recovers after severe latency clears" - target: all + description: "Block production continues on reference node (BN3) after latency clears — BN1 stream may not recover at this severity tier" + target: block-node-3 args: wait_seconds: 60 max_attempts: 5 - id: block-rate-floor type: block-rate-floor - description: "Post-chaos block rate recovers to ≥ 0.1 blk/s (recovery, not steady-state)" - target: all + description: "Post-chaos block rate on reference node (BN3) is ≥ 0.1 blk/s — confirms network liveness even if BN1 stream was severed" + target: block-node-3 args: min_rate_per_sec: 0.1 - window_seconds: 30 + window_seconds: 30 \ No newline at end of file diff --git a/tools-and-tests/scripts/solo-e2e-test/tests/latency-stress.yaml b/tools-and-tests/scripts/solo-e2e-test/tests/latency-stress.yaml index 7829a53085..a17137ce41 100644 --- a/tools-and-tests/scripts/solo-e2e-test/tests/latency-stress.yaml +++ b/tools-and-tests/scripts/solo-e2e-test/tests/latency-stress.yaml @@ -1,30 +1,48 @@ # SPDX-License-Identifier: Apache-2.0 # -# Heavy ("stress") CN↔CN + BN↔BN + CN↔BN Latency Test +# Stress-Level CN↔BN Latency — 1/3 BN Degradation Test # -# A deliberately harsher sibling of latency-all-three: ~3x the baseline delays, -# wider jitter, and correlation (bursty delays) to push the network past the -# "no visible impact" regime into "degrades but recovers": -# - CN ↔ CN : 300 ms ± 80 ms, correlation 50% (bidirectional) -# - CN ↔ BN : 450 ms ± 120 ms, correlation 50% (bidirectional) -# - BN ↔ BN : 600 ms ± 150 ms, correlation 50% (bidirectional) +# One of three BNs (BN1) receives moderate-to-severe CN↔BN and BN→BN latency; +# the other two BNs run at normal latency. # -# Intent: under this load the BNs should fall behind and recover via backfill, -# then resume normal block production once the latency clears. The assertions run -# AFTER the chaos is cleared, so they verify graceful recovery (not steady-state -# throughput under chaos): nodes healthy, blocks increasing again, backfill was -# exercised, and a (reduced) post-chaos block-rate floor still holds. +# BN1 — "approaching threshold" : CN↔BN1 5000 ms ± 1250 ms + 50% loss (corr 50%), BN1→BN 2500 ms (direction=to) +# BN2 — "normal" : no chaos +# BN3 — "normal" : no chaos +# CNs — global light : CN↔CN 100 ms ± 20 ms +# +# At 5000ms one-way + 50% loss: TCP throughput (~100-250 KB/s) far exceeds the +# block data rate (~2 KB/block). The delay is constant (~2.5 blocks in-flight), +# not cumulative — BN1 keeps up rather than falling progressively behind. Block +# divergence only becomes observable at the severe tier where 8000ms + 75% +# correlated loss triggers burst-loss RTO chains that freeze the stream. +# +# TCP stays alive: kernel retry limit (~34 min at 5000ms RTT) far exceeds the +# test window. gRPC keepalive is disabled by default on grpc-java (CN side). +# After chaos clears, the CN's buffered backlog drains at full speed; BN1 catches +# up within seconds. +# +# BN→BN uses direction=to (unidirectional) to avoid double-stacking latency. +# +# Intent: survival test — validates that BN1 remains healthy, keeps receiving +# blocks, and converges after chaos clears under high-latency conditions. Block +# divergence is NOT asserted at this tier; at this loss/latency level TCP +# throughput still exceeds the block data rate so lag is constant, not +# cumulative. A measurable application-level impact assertion requires bandwidth +# throttling (see https://github.com/hiero-ledger/hiero-block-node/issues/3379). +# Use latency-severe for divergence assertions. +# +# Requires topology with ≥ 3 BNs (e.g. paired-3, 7cn-3bn-distributed). +# The dryrun check will fail on 3cn-1bn. # # Pre-req: -# task up TOPOLOGY=paired-3 # TSS is on by default (now supported) +# task up TOPOLOGY=paired-3 # CHAOS_ENABLED=true task chaos:install -# (TSS is on by default; CN_VERSION=latest is min-enforced to a TSS-capable tag.) # # Usage: # CHAOS_ENABLED=true TOPOLOGY=paired-3 task test:run TEST_FILE=tests/latency-stress.yaml name: latency-stress -description: "Heavy CN/BN bidirectional latency (3x baseline, bursty) — degrade-and-recover" +description: "CN↔BN severe latency on 1/3 of BNs (BN1) — single-node degradation & greedy backfill" events: - id: load-start @@ -38,42 +56,42 @@ events: - id: inject-cn-cn type: inject-latency - description: "Inject CN↔CN 300ms ± 80ms (corr 50%)" + description: "Inject CN↔CN 100ms ± 20ms — light global CN background latency" delay: 30 args: name: cn-to-cn source: { kind: network-node } target: { kind: network-node } - latency: 300ms - jitter: 80ms - correlation: "50" + latency: 100ms + jitter: 20ms bidirectional: true - - id: inject-bn-bn + - id: inject-cn-bn1 type: inject-latency - description: "Inject BN↔BN 600ms ± 150ms (corr 50%)" + description: "Inject CN↔BN1 5000ms ± 1250ms + 50% loss (corr 50%) — approaching-threshold latency" delay: 35 args: - name: bn-to-bn - source: { kind: block-node } - target: { kind: block-node } - latency: 600ms - jitter: 150ms + name: cn-to-bn1 + source: { kind: network-node } + target: { kind: block-node, name: block-node-1 } + latency: 5000ms + jitter: 1250ms correlation: "50" bidirectional: true + loss: "50%" - - id: inject-cn-bn + - id: inject-bn1-bn type: inject-latency - description: "Inject CN↔BN 450ms ± 120ms (corr 50%)" + description: "Inject BN1→BN peers 2500ms ± 625ms (direction=to) — BN1 slow peer replication" delay: 40 args: - name: cn-to-bn - source: { kind: network-node } + name: bn1-to-bn + source: { kind: block-node, name: block-node-1 } target: { kind: block-node } - latency: 450ms - jitter: 120ms + latency: 2500ms + jitter: 625ms correlation: "50" - bidirectional: true + bidirectional: false - id: clear-cn-cn type: clear-latency @@ -81,17 +99,17 @@ events: delay: 330 args: { name: cn-to-cn } - - id: clear-bn-bn + - id: clear-cn-bn1 type: clear-latency - description: "Clear BN↔BN" + description: "Clear CN↔BN1" delay: 332 - args: { name: bn-to-bn } + args: { name: cn-to-bn1 } - - id: clear-cn-bn + - id: clear-bn1-bn type: clear-latency - description: "Clear CN↔BN" + description: "Clear BN1→BN peers" delay: 334 - args: { name: cn-to-bn } + args: { name: bn1-to-bn } - id: load-stop type: load-stop @@ -101,36 +119,34 @@ events: assertions: - id: all-healthy type: node-healthy - description: "All BNs healthy after heavy stacked chaos" + description: "All BNs remain healthy despite BN1 severe latency" target: all - id: all-have-blocks type: block-available - description: "All BNs have blocks" + description: "All BNs have blocks after chaos clears" target: all args: min_block: 0 - - id: backfill-triggered - type: backfill-triggered - description: "BNs fell behind under heavy latency and caught up via backfill" - target: all + - id: blocks-converged + type: blocks-converged + description: "All BNs converge to within 5 blocks of each other after latency clears — proves CN↔BN streams survived and BN1 caught up" args: - grep: "backfill" - since_seconds: 420 + tolerance_blocks: 5 - id: blocks-increasing type: blocks-increasing - description: "Block production resumes after the latency clears" + description: "Block production recovers on all BNs after latency clears" target: all args: - wait_seconds: 45 - max_attempts: 4 + wait_seconds: 60 + max_attempts: 5 - id: block-rate-floor type: block-rate-floor - description: "Post-chaos block rate recovers to ≥ 0.2 blk/s" + description: "Post-chaos block rate recovers to ≥ 0.1 blk/s on all BNs" target: all args: - min_rate_per_sec: 0.2 + min_rate_per_sec: 0.1 window_seconds: 30