Performance Harness #90
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Harness | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 3 * * *" | |
| push: | |
| branches: [main] | |
| paths: | |
| - "src/**" | |
| - "tests/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - ".github/workflows/performance-harness.yml" | |
| concurrency: | |
| group: performance-harness-${{ github.ref }} | |
| cancel-in-progress: true | |
| # Deny-all token default. This workflow only reads repository contents and | |
| # uploads benchmark artifacts produced in-run. | |
| permissions: {} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| performance: | |
| name: Extreme-load performance harness | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 35 | |
| # Preserve a typed release target that the native Linux release matrix can | |
| # reuse. CARGO_BUILD_TARGET gives it an explicit nested target triple. | |
| env: | |
| CARGO_TARGET_DIR: target/ci-release | |
| CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust toolchain | |
| run: rustup show | |
| # Release artifacts are structurally isolated from debug and profiling. | |
| # Only trusted pushes to main save; scheduled and manually dispatched runs | |
| # may restore but can never publish a cache entry. | |
| - name: Cache Rust release dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v1-serval-rust | |
| shared-key: ci-release-x86_64-unknown-linux-gnu | |
| workspaces: . -> target/ci-release | |
| cache-bin: false | |
| cache-workspace-crates: false | |
| save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Tune kernel for high-concurrency sockets | |
| run: | | |
| # ── File descriptors ──────────────────────────────────────────────── | |
| # Each TCP connection consumes one fd. At c25733 the harness needs | |
| # 25K+ fds in a single process. ulimit -n cannot raise the hard limit | |
| # in a GitHub Actions shell (EPERM), so use prlimit which works when | |
| # invoked with sudo and can set both soft and hard limits directly. | |
| sudo prlimit --pid $$ --nofile=1048576:1048576 | |
| # ── Port range & TIME_WAIT pool ────────────────────────────────────── | |
| # Expand ephemeral ports from ~28K (32768–60999) to ~64K so that | |
| # 25K+ simultaneous outbound connections never exhaust the pool. | |
| sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535" | |
| # Allow TIME_WAIT sockets to be reused when tcp_timestamps is on | |
| # (default). Prevents port exhaustion from accumulated TIME_WAIT | |
| # sockets between stages. | |
| sudo sysctl -w net.ipv4.tcp_tw_reuse=1 | |
| # Increase the TIME_WAIT bucket count (default 131072) so the kernel | |
| # keeps TIME_WAIT state properly rather than silently destroying | |
| # sockets and making RST responses look like transport errors. | |
| sudo sysctl -w net.ipv4.tcp_max_tw_buckets=2000000 | |
| # Drain FIN_WAIT2 sockets faster between stages (default 60s; | |
| # 15s is safe for loopback-only test traffic). | |
| sudo sysctl -w net.ipv4.tcp_fin_timeout=15 | |
| # ── Listen backlog ─────────────────────────────────────────────────── | |
| # Raise the kernel ceiling to match the backlog the delivery server | |
| # requests, so SYNs are not dropped at the kernel queue before hyper | |
| # sees them during burst connection storms. | |
| sudo sysctl -w net.core.somaxconn=65535 | |
| sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535 | |
| # Increase the per-interface receive queue so packets are not dropped | |
| # when NIC (loopback) delivers faster than the kernel can process. | |
| sudo sysctl -w net.core.netdev_max_backlog=65535 | |
| # ── Socket buffer sizes ────────────────────────────────────────────── | |
| # Raise the ceiling for socket read/write buffers (default 212992 B). | |
| # The kernel autotuner (tcp_rmem/tcp_wmem) stays within this ceiling; | |
| # without raising it, high-throughput loopback connections cannot grow | |
| # their windows enough to fully saturate the pipe. | |
| sudo sysctl -w net.core.rmem_max=134217728 | |
| sudo sysctl -w net.core.wmem_max=134217728 | |
| # TCP buffer autotuning bounds: min / default / max. | |
| # max matches the ceiling above; default is kept modest to avoid | |
| # wasting memory on idle connections. | |
| sudo sysctl -w net.ipv4.tcp_rmem="4096 131072 134217728" | |
| sudo sysctl -w net.ipv4.tcp_wmem="4096 131072 134217728" | |
| # ── Congestion control & idle behaviour ────────────────────────────── | |
| # Don't reduce the congestion window after an idle period. Between | |
| # stages the load generators are idle for a brief moment; without | |
| # this, TCP slow-start fires at the beginning of each stage and | |
| # inflates early-stage latency, biasing p50/p95 upward. | |
| sudo sysctl -w net.ipv4.tcp_slow_start_after_idle=0 | |
| # Switch to BBR congestion control. CUBIC is loss-based: it halves | |
| # its window on any detected loss, which on loopback (zero true | |
| # congestion) means even transient tail drops trigger unnecessary | |
| # window reductions and latency spikes. BBR is model-based and | |
| # estimates bandwidth/RTT directly, producing much smoother p95/p99 | |
| # under burst storms. fq is the paired qdisc BBR requires. | |
| sudo sysctl -w net.core.default_qdisc=fq | |
| sudo sysctl -w net.ipv4.tcp_congestion_control=bbr | |
| # Disable TCP metrics caching. The kernel caches ssthresh and RTT | |
| # estimates per destination IP across connections. In the harness, | |
| # the slow early stages (high latency, depressed throughput) leave | |
| # pessimistic cached values that throttle the window size at the | |
| # start of later, faster stages — polluting inter-stage comparisons. | |
| sudo sysctl -w net.ipv4.tcp_no_metrics_save=1 | |
| # Enable TCP Fast Open for both client (1) and server (2) sides. | |
| # Eliminates the SYN round-trip for repeated connections to the same | |
| # address, which is exactly the pattern the harness creates. | |
| sudo sysctl -w net.ipv4.tcp_fastopen=3 | |
| # ── Connection teardown & dead-port reclamation ────────────────────── | |
| # Keepalive: start probing idle connections after 10s, every 2s, | |
| # declaring dead after 3 failed probes (16s total vs 2.5h default). | |
| # Trade-off: ~3 extra TCP segments per idle pooled connection per 16s | |
| # — negligible on loopback. Benefit: dead connections and their ports | |
| # are detected and freed quickly between stages. | |
| sudo sysctl -w net.ipv4.tcp_keepalive_time=10 | |
| sudo sysctl -w net.ipv4.tcp_keepalive_intvl=2 | |
| sudo sysctl -w net.ipv4.tcp_keepalive_probes=3 | |
| # Orphan retries: when the client close()s a socket (e.g. after a | |
| # request timeout fires), the kernel keeps a FIN_WAIT1 orphan until | |
| # the remote ACKs the FIN. The default retry count (~8) can hold | |
| # these orphans for ~50s. Setting 2 reclaims them in ~7s. | |
| # Trade-off: would RST a FIN whose ACK takes >7s, impossible on | |
| # loopback where the server is co-located. | |
| sudo sysctl -w net.ipv4.tcp_orphan_retries=2 | |
| # SYN retries: default 6 means ~127s before giving up on a dropped | |
| # SYN. With our large SYN queue (65535) SYN drops are rare, but when | |
| # they do occur at the stage collapse boundary we want fast failure. | |
| # Setting 3 limits the SYN timeout to ~15s. | |
| # Trade-off: if the server recovers from a SYN-drop within 15s, | |
| # the connection would have succeeded with the default. Acceptable — | |
| # a recovered server at that concurrency still produces inflated | |
| # latencies that health-gate logic will classify as unhealthy anyway. | |
| sudo sysctl -w net.ipv4.tcp_syn_retries=3 | |
| - name: Run performance harness | |
| run: cargo test --release --features integration --test performance_harness -- --ignored --nocapture | |
| env: | |
| SERVAL_SKIP_FRONTEND_BUILD: "1" | |
| PERF_RESULTS_PATH: target/performance-harness-report.json | |
| PERF_REQUEST_TIMEOUT_SECS: "15" | |
| PERF_LATENCY_SAMPLE_STRIDE: "24" | |
| PERF_LATENCY_SAMPLE_CAPACITY: "250000" | |
| PERF_PEAK_INITIAL_CONCURRENCY: "256" | |
| PERF_PEAK_MAX_CONCURRENCY: "16384" | |
| PERF_PEAK_STAGE_DURATION_SECS: "8" | |
| PERF_PEAK_MULTIPLIER_PCT: "160" | |
| PERF_PEAK_CONTINUE_MAX_ERROR_PCT: "4" | |
| PERF_PEAK_CONTINUE_MAX_P95_MS: "1200" | |
| PERF_PEAK_MAX_UNHEALTHY_STAGES: "2" | |
| PERF_ADVERSARIAL_LEGIT_CONCURRENCY: "64" | |
| PERF_ADVERSARIAL_FORGED_INITIAL_CONCURRENCY: "512" | |
| PERF_ADVERSARIAL_FORGED_MAX_CONCURRENCY: "32768" | |
| PERF_ADVERSARIAL_FORGED_MULTIPLIER_PCT: "175" | |
| PERF_ADVERSARIAL_STAGE_DURATION_SECS: "8" | |
| PERF_ADVERSARIAL_WRITER_INTERVAL_MS: "120" | |
| PERF_ADVERSARIAL_CONTINUE_MIN_LEGIT_RPS: "150" | |
| PERF_ADVERSARIAL_CONTINUE_MAX_ERROR_PCT: "8" | |
| PERF_ADVERSARIAL_CONTINUE_MAX_P95_MS: "1600" | |
| PERF_ADVERSARIAL_CONTINUE_MIN_FORGED_REJECTION_PCT: "98" | |
| PERF_ADVERSARIAL_CONTINUE_MIN_CONTROL_WRITE_SUCCESS_PCT: "85" | |
| PERF_ADVERSARIAL_MAX_UNHEALTHY_STAGES: "2" | |
| PERF_ASSERT_PEAK_MIN_TESTED_CONCURRENCY: "2048" | |
| PERF_ASSERT_PEAK_MIN_BEST_RPS: "900" | |
| PERF_ASSERT_PEAK_MAX_BEST_ERROR_PCT: "8" | |
| PERF_ASSERT_ADVERSARIAL_MIN_TESTED_FORGED_CONCURRENCY: "4096" | |
| PERF_ASSERT_ADVERSARIAL_MIN_LEGIT_RPS: "150" | |
| PERF_ASSERT_ADVERSARIAL_MAX_LEGIT_ERROR_PCT: "15" | |
| PERF_ASSERT_FORGED_REJECTION_PCT: "99" | |
| PERF_ASSERT_CONTROL_WRITE_SUCCESS_PCT: "80" | |
| - name: Generate step summary | |
| if: always() | |
| run: | | |
| REPORT=target/performance-harness-report.json | |
| if [ ! -f "$REPORT" ]; then | |
| echo '> No performance report produced (harness did not complete).' >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| jq -r ' | |
| "## Peak Traffic Profile\n", | |
| "| Concurrency | RPS | p50 ms | p95 ms | p99 ms | Errors | Status |", | |
| "|---:|---:|---:|---:|---:|---:|:---:|", | |
| (.peak_stages[] | | |
| "| " + (.concurrency|tostring) | |
| + " | " + (.snapshot.throughput_rps * 10 | round / 10 | tostring) | |
| + " | " + (.snapshot.p50_ms|tostring) | |
| + " | " + (.snapshot.p95_ms|tostring) | |
| + " | " + (.snapshot.p99_ms|tostring) | |
| + " | " + (.snapshot.error_rate * 10000 | round / 100 | tostring) + "%" | |
| + " | " + (if .healthy then "✅" else "❌" end) + " |" | |
| ), | |
| "\n**Best:** `" + (.peak_best.concurrency|tostring) + "` concurrency → **" | |
| + (.peak_best.snapshot.throughput_rps * 10 | round / 10 | tostring) | |
| + " rps** · p95 " + (.peak_best.snapshot.p95_ms|tostring) + " ms\n", | |
| "---\n", | |
| "## Adversarial Profile\n", | |
| "| Forged conc. | Legit RPS | Legit p95 ms | Legit errors | Forged rejection | CP writes ok | Status |", | |
| "|---:|---:|---:|---:|---:|---:|:---:|", | |
| (.adversarial_stages[] | | |
| "| " + (.forged_concurrency|tostring) | |
| + " | " + (.legit.throughput_rps * 10 | round / 10 | tostring) | |
| + " | " + (.legit.p95_ms|tostring) | |
| + " | " + (.legit.error_rate * 10000 | round / 100 | tostring) + "%" | |
| + " | " + (.forged_rejection_rate * 10000 | round / 100 | tostring) + "%" | |
| + " | " + (.control_write_success_rate * 100 | round | tostring) + "%" | |
| + " | " + (if .healthy then "✅" else "❌" end) + " |" | |
| ), | |
| "\n**Best:** `" + (.adversarial_best.forged_concurrency|tostring) | |
| + "` forged concurrency → legit **" | |
| + (.adversarial_best.legit.throughput_rps * 10 | round / 10 | tostring) | |
| + " rps** · forged rejection **" | |
| + (.adversarial_best.forged_rejection_rate * 10000 | round / 100 | tostring) + "%**" | |
| ' "$REPORT" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload performance report | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: performance-harness-report | |
| path: target/performance-harness-report.json | |
| if-no-files-found: warn | |
| flamegraphs: | |
| name: Representative hot-path flamegraphs | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| # Keep symbol-bearing profiling output physically and by cache key separate | |
| # from stripped release and debug build artifacts. | |
| env: | |
| CARGO_TARGET_DIR: target/ci-profiling | |
| RUSTFLAGS: "-C force-frame-pointers=yes" | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Rust toolchain | |
| run: rustup show | |
| # RUSTFLAGS is job-scoped so rust-cache includes the frame-pointer setting | |
| # in its environment hash as well as using this separate target root. | |
| # Cache writes remain exclusive to trusted main pushes. | |
| - name: Cache Rust profiling dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v1-serval-rust | |
| shared-key: ci-profiling | |
| workspaces: . -> target/ci-profiling | |
| cache-bin: false | |
| cache-workspace-crates: false | |
| save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Capture representative flamegraphs | |
| run: >- | |
| cargo test --profile profiling --locked | |
| --features integration,profiling | |
| --test performance_harness | |
| representative_hot_path_flamegraphs | |
| -- --ignored --nocapture | |
| env: | |
| SERVAL_SKIP_FRONTEND_BUILD: "1" | |
| PERF_REQUEST_TIMEOUT_SECS: "15" | |
| PERF_LATENCY_SAMPLE_STRIDE: "24" | |
| PERF_LATENCY_SAMPLE_CAPACITY: "250000" | |
| PERF_FLAMEGRAPH_OUTPUT_DIR: target/flamegraphs | |
| PERF_FLAMEGRAPH_FREQUENCY_HZ: "99" | |
| PERF_FLAMEGRAPH_DURATION_SECS: "15" | |
| PERF_FLAMEGRAPH_PEAK_CONCURRENCY: "512" | |
| PERF_FLAMEGRAPH_ADVERSARIAL_FORGED_CONCURRENCY: "2048" | |
| PERF_FLAMEGRAPH_ADVERSARIAL_LEGIT_CONCURRENCY: "64" | |
| PERF_FLAMEGRAPH_WRITER_INTERVAL_MS: "120" | |
| - name: Validate flamegraphs | |
| if: always() | |
| run: | | |
| for graph in target/flamegraphs/peak.svg target/flamegraphs/adversarial.svg; do | |
| test -s "$graph" | |
| grep -q '<svg' "$graph" | |
| done | |
| - name: Upload flamegraphs | |
| id: flamegraph-artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: hot-path-flamegraphs | |
| path: | | |
| target/flamegraphs/peak.svg | |
| target/flamegraphs/adversarial.svg | |
| if-no-files-found: error | |
| retention-days: 14 | |
| - name: Generate flamegraph summary | |
| if: always() | |
| env: | |
| ARTIFACT_URL: ${{ steps.flamegraph-artifact.outputs.artifact-url }} | |
| run: | | |
| { | |
| echo '## Hot-Path CPU Flamegraphs' | |
| echo | |
| if [ -n "$ARTIFACT_URL" ]; then | |
| echo "[Download the interactive Peak and Adversarial SVGs]($ARTIFACT_URL)" | |
| else | |
| echo '> Flamegraph artifacts were not produced. Inspect the profiling step above.' | |
| fi | |
| echo | |
| echo '| Scenario | Representative workload | Sampling |' | |
| echo '|---|---|---|' | |
| echo '| Peak | 512 concurrent rendered, cache-hot Data Plane reads | 15s at 99 Hz |' | |
| echo '| Adversarial | 2,048 forged + 64 legitimate readers + writes every 120ms | 15s at 99 Hz |' | |
| echo | |
| echo 'Open an SVG in a browser, hover for sample share, and click frames to zoom.' | |
| echo 'Search for **serval::delivery**, **IdSigner::verify**, cache, renderer, and database frames.' | |
| echo | |
| echo '> Profiling is diagnostic. The separate stripped-release harness remains the SLO authority.' | |
| } >> "$GITHUB_STEP_SUMMARY" |