From 6d64c0ac608defbfb179933aea5c110fc0b246ab Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 06:31:47 +0000 Subject: [PATCH 01/30] HiCache sizing fixes (ratio vs size), NCCL/image updates, disable prefill cudagraph for DSv4 agentic disagg Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 16 +++++- benchmarks/multi_node/amd_utils/env.sh | 20 +++++++ benchmarks/multi_node/amd_utils/job.slurm | 1 + benchmarks/multi_node/amd_utils/models.yaml | 9 +-- .../multi_node/amd_utils/server_sglang.sh | 10 +++- configs/amd-master.yaml | 57 ++++++++++--------- 6 files changed, 78 insertions(+), 35 deletions(-) diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index e5dde8d120..efdf320119 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -83,8 +83,20 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" - # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. - export HICACHE_RATIO="${HICACHE_RATIO:-}" + # HiCache ratio (host pool = ratio * GPU KV pool). + export HICACHE_RATIO="${HICACHE_RATIO:-4}" + # server_sglang.sh prefers an absolute --hicache-size (derived from + # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over + # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the + # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the + # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself: + # that var is also the shared client-side gate (benchmark_lib.sh requires + # it to be a positive integer whenever KV_OFFLOADING=dram) and gets + # forwarded into the aiperf sibling container's client.env, so unsetting + # it here made the client fail its own env validation before benchmarking + # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB + # capacity") even though the servers came up fine. + export FORCE_HICACHE_RATIO=1 # ── HiCache layout/backend by tier ── # L3 (Mooncake): page_first + direct + write_through + storage=mooncake diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index baa917b449..2549e63af2 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -275,6 +275,26 @@ else # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} + # Decode CUDA-graph capture crash on ROCm 7.2.0 (TP8+EP8, mori a2a). + # Symptom: during decode cuda-graph capture, the torch ProcessGroupNCCL + # *watchdog* thread calls hipEventQuery() to poll in-flight NCCL work. + # ROCm <= 7.2.0's HIP runtime does NOT honor cudaStreamCaptureModeThreadLocal, + # so the watchdog's cross-thread query touches the main thread's active + # capture and invalidates it -> "HIP error: operation not permitted on an + # event last recorded in a capturing stream (hipErrorCapturedEvent)" -> + # watchdog aborts -> "Rank 0 scheduler died during initialization" (-6). + # This is a HIP runtime bug, not OOM and not a mori/deepep-mode bug (it + # fires for --deepep-mode normal and auto alike; EP8+mori just adds NCCL + # PGs that make the watchdog race fire). Refs: sgl-project/sglang#29235, + # #24011; ROCm/hip#3876; pytorch/pytorch#176251. + # Real fix = ROCm 7.2.2+ (honors THREAD_LOCAL). Until the base image is + # bumped, TORCH_NCCL_BLOCKING_WAIT=true makes NCCL work completion use a + # blocking wait instead of the async watchdog hipEventQuery poll, so no + # event is queried during capture. CUDA graph stays fully enabled. + export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}" + export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" + export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" + # ========================================================================= # DeepSeek-V4-Pro PD recipe overrides # Placed at the end of the SGLang env block so it wins over the global diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 863d9d7635..5a59abeea5 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -494,6 +494,7 @@ fi HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" cat > "$HICACHE_MC_CONFIG" <=16 queue-saturation / decode-stall failure mode). @@ -388,6 +388,7 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 decode: mem_fraction_static: 0.85 + disable_cuda_graph: false prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 4e917fd26e..d37171fe5c 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -537,6 +537,12 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then # per-node DRAM budget computed by the sweep generator (enforcement); fall # back to --hicache-ratio (relative to the GPU KV pool) when no budget is # provided, keeping configs that predate the budget unchanged. + # FORCE_HICACHE_RATIO lets a recipe opt into ratio-based sizing without + # unsetting TOTAL_CPU_DRAM_GB — that var is also the shared client-side + # gate (benchmark_lib.sh requires it whenever KV_OFFLOADING=dram) and is + # forwarded verbatim into client.env below, so unsetting it here would + # make the aiperf client container fail its own env validation before + # ever sending a request. HICACHE_RATIO="${HICACHE_RATIO:-5}" HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" # DeepSeek V4's hybrid HiCache pool rejects --hicache-size (requires @@ -544,7 +550,9 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then # See sglang _deepseek_v4_num_host_pages() (raises ValueError when # server_args.hicache_size > 0): # https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 - if [[ -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then + # FORCE_HICACHE_RATIO additionally lets a recipe opt into ratio-based sizing + # for any other model without unsetting TOTAL_CPU_DRAM_GB (see comment above). + if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill # offloads KV to CPU DRAM today); --hicache-size is per rank per host # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 8a855d21e1..e2b8ac2b53 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2248,7 +2248,7 @@ minimaxm3-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -2278,7 +2278,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: # upstream CI can fetch it; aiperf is built on the fly from # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) # selects the sibling-container mode. - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" decode: num-worker: 1 tp: 8 @@ -2287,32 +2287,32 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - # - spec-decoding: "none" - # conc-list: [ 64, 128 ] - # kv-offloading: dram - # kv-offload-backend: { name: hicache } - # prefill: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: true - # additional-settings: - # - "PREFILL_NODES=1" - # # Node-0 sibling client: run the aiperf trace-replay in its own sibling - # # container on node 0 (via the host docker socket) instead of inside the - # # server container. Reuse the (publicly pullable) server image so - # # upstream CI can fetch it; aiperf is built on the fly from - # # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) - # # selects the sibling-container mode. - # - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" - # decode: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: true - # additional-settings: - # - "DECODE_NODES=1" - # - "DECODE_MTP_SIZE=0" + - spec-decoding: "none" + conc-list: [ 64, 96, 128 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" # DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. # EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no @@ -2363,3 +2363,4 @@ dsv4-fp8-mi325x-vllm-mtp: # is 55.8%. conc128 passed cleanly on the memory-tightest SKU (MI300X); # cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512). - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } + From cef101d81e37df0a67acf0fecba208045cf41e8a Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 06:31:48 +0000 Subject: [PATCH 02/30] Update DPA config, add RDMA/GPU pre-flight gates to job.slurm, and fix router policy env var/name Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 2 +- .../amd_utils/helpers/gpu_sanity.sh | 38 +++++ .../amd_utils/helpers/rdma_check.sh | 160 ++++++++++++++++++ benchmarks/multi_node/amd_utils/job.slurm | 29 ++++ .../multi_node/amd_utils/server_sglang.sh | 15 +- .../multi_node/amd_utils/trace_replay.sh | 6 + 6 files changed, 244 insertions(+), 6 deletions(-) create mode 100755 benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh create mode 100755 benchmarks/multi_node/amd_utils/helpers/rdma_check.sh diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index efdf320119..f43c81e1c6 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -134,7 +134,7 @@ export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" # ── SGLang PD router policy + server metrics ── -export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" export ENABLE_METRICS="${ENABLE_METRICS:-1}" # ── MTP ── diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh new file mode 100755 index 0000000000..9e3df3b3be --- /dev/null +++ b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# GPU sanity gate, run on each node AFTER all containers are stopped. +# +# At that point VRAM should fall back to driver baseline. Any large remaining +# usage means a bare (non-containerized) process is hogging the GPU -- which +# would otherwise surface as an opaque "memory capacity is unbalanced" OOM in +# the middle of model load ~30 min into the job. We fail fast and name the +# offender instead. We only report; we never kill another user's process. +# +# Ported from https://github.com/ROCm/InferenceY/blob/main/benchmarks/multi_node/amd_utils/gpu_sanity.sh +set -uo pipefail + +# Fail the node if any single GPU still has >= this many GB used. +GPU_BUSY_GB="${GPU_BUSY_GB:-8}" +# Poll a few times to give docker stop time to release VRAM before judging. +GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-6}" + +if ! command -v rocm-smi >/dev/null 2>&1; then + echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 + exit 0 +fi + +busy_gb=0 +for _ in $(seq "$GPU_DRAIN_TRIES"); do + used=$(rocm-smi --showmeminfo vram --json 2>/dev/null \ + | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \ + | grep -oE '[0-9]+' | sort -n | tail -1) + busy_gb=$(( ${used:-0} / 1024 / 1024 / 1024 )) + if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then + exit 0 + fi + sleep 5 +done + +echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 +echo " (threshold ${GPU_BUSY_GB}GB) -- a bare (non-containerized) process is hogging the GPU:" >&2 +rocm-smi --showpids >&2 || true +exit 1 diff --git a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh new file mode 100755 index 0000000000..86acabbab0 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Pre-flight RDMA QoS/DCQCN validation for a single node, run by job.slurm via +# `srun` across every allocated node BEFORE any container/GPU time is spent. +# +# A misconfigured NIC (PFC not covering the RoCE priority, DCQCN disabled, ...) +# doesn't make MoRI's cross-node EP/RDMA transfers error out -- it just quietly +# degrades, and the only symptom is unexplained tail latency or throughput +# variance in the benchmark numbers hours later. Failing fast here, before the +# job burns node-hours, is much cheaper than debugging that after the fact. +# +# check_qos()/check_dcqcn() below are trimmed/adapted from ROCm/mori's +# tools/env_check.sh: +# https://github.com/ROCm/mori/blob/main/tools/env_check.sh +# The upstream script's expensive ib_write_bw/ib_write_lat full-mesh bandwidth +# and latency tests are intentionally NOT ported here -- this runs before +# every single job, so it must be fast (seconds), not a multi-minute +# fabric-wide benchmark of its own. +# +# Scoped to AMD Pollara (ionic) NICs via nicctl, matching this repo's current +# fleet (see env.sh's nicctl-based MORI_RDMA_TC/SL detection). If bnxt_re or +# mlx5 NICs are added to the fleet, port the bnxt_*/mlx5_* check_* functions +# from the upstream script following the same pattern. +# +# Exit code: 0 = OK (or gracefully skipped, e.g. no ionic NICs on this host), +# 1 = hard QoS/DCQCN misconfiguration -- do not proceed with the run. +set -uo pipefail + +AINIC_MIN_VER="1.117.5-a-45" # minimum recommended AINIC firmware for IBGDA + +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' + +log_ok() { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC} $*"; } +log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; } +log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; } +die() { log_fail "$@"; exit 1; } + +# version_ge -> true if >= (dotted/hyphenated, via sort -V) +version_ge() { + local cand="$1" min="$2" + [[ "$cand" == "$min" ]] && return 0 + [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]] +} + +# check_ainic_version_recommendation +# warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the +# recommended minimum for cross-node MORI (EP over RDMA / IBGDA). +check_ainic_version_recommendation() { + local ver="$1" + [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; } + if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then + log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER" + elif version_ge "$ver" "$AINIC_MIN_VER"; then + log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" + else + log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" + fi +} + +# check_versions() -- informational only (never hard-fails the job). +check_versions() { + local fw_output sw_output + fw_output=$(sudo nicctl show version firmware 2>/dev/null) + sw_output=$(sudo nicctl show version host-software 2>/dev/null) + + local fw_versions fw_count + fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u) + fw_count=$(echo "$fw_versions" | grep -c . || true) + if [[ "$fw_count" -ne 1 ]]; then + log_warn "firmware versions not consistent across NICs:" + echo "$fw_versions" + local v + while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions" + else + log_ok "firmware : $fw_versions" + check_ainic_version_recommendation "$fw_versions" + fi + + local nicctl_ver + nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}') + [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version" +} + +# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop +# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise. +check_qos() { + local qos_output + qos_output=$(sudo nicctl show qos 2>/dev/null) + [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing" + + local class_type + class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}') + [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'" + log_ok "classification type : DSCP" + + local nd_prio_raw + nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}') + [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority" + local nd_prios=() + IFS=',' read -ra nd_prios <<< "$nd_prio_raw" + log_ok "no-drop priorities : ${nd_prios[*]}" + + local pfc_bitmap + pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}') + [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)" + local p + for p in "${nd_prios[@]}"; do + (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p" + done + log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)" +} + +# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and +# the CNP DSCP must be consistent across NICs. Exits 1 otherwise. +check_dcqcn() { + local dcqcn_output + dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null) + [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing" + + local total + total=$(echo "$dcqcn_output" | grep -c "ROCE device") + [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output" + + local disabled + disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true) + if [[ -n "$disabled" ]]; then + log_fail "some ROCE devices have DCQCN disabled:" + echo "$disabled" + exit 1 + fi + log_ok "DCQCN enabled on all $total ROCE devices" + + local cnp_values cnp_count + cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u) + cnp_count=$(echo "$cnp_values" | grep -c . || true) + [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values" + log_ok "CNP DSCP = $cnp_values (consistent across all NICs)" +} + +# ============================= main ============================= + +if ! command -v nicctl &>/dev/null; then + log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)" + exit 0 +fi + +# nicctl exits 0 even with no NIC present, so check its output rather than its exit code. +_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true) +if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then + log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check" + exit 0 +fi + +check_versions +check_qos +check_dcqcn + +log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 5a59abeea5..3c86f6fa06 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -345,6 +345,29 @@ export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) +# ============================================================================= +# RDMA QoS / DCQCN Pre-flight Check +# ============================================================================= +# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before +# any container/GPU time is spent. Runs on the bare host (nicctl is a host +# tool). See rdma_check.sh for details. +RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" +if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then + echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" +elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then + echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." + srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" + RDMA_CHECK_RC=$? + if [[ $RDMA_CHECK_RC -ne 0 ]]; then + echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." + echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." + exit 1 + fi + echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" +else + echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" +fi + cleanup() { echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` @@ -636,6 +659,12 @@ fi # end: if ENGINE == atom-disagg \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true +# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare +# (non-containerized) process hogging the GPU -- fail fast (and name it) +# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit +# tears down the whole job on non-zero exit. +bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" + # Start vLLM external router container on node 0 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index d37171fe5c..f1124e5205 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -791,12 +791,17 @@ if [ "$NODE_RANK" -eq 0 ]; then # across the agentic trace; round_robin decode keeps the single decode worker # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS. ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" - ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + # server_sglang.sh previously read ROUTER_PREFILL_POLICY, but the recipe + # scripts export PREFILL_ROUTER_POLICY, so the recipe's policy override was + # silently ignored and the router always fell back to this hardcoded + # default. Also comment out ROUTER_DECODE_POLICY for now (superseded by + # --dp-aware below). + ROUTER_PREFILL_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" + # ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --dp-aware --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" else # DI router config (8k1k branch, run 28696443568): with defaults the per-worker # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial. @@ -990,8 +995,8 @@ if [ "$NODE_RANK" -eq 0 ]; then ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ DISAGG IS_MULTINODE \ TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_ENABLE_DP PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_ENABLE_DP DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh index b3eb0b8fd7..f4e9f84c23 100644 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -107,6 +107,12 @@ RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" mkdir -p "$RESULT_DIR" +if [ "$PREFILL_ENABLE_DP" = "true" ]; then + set -x + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + set +x +fi + resolve_trace_source install_agentic_deps From c7a5d9d202dc9040f39124601340247e9d38e89c Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 08:00:51 +0000 Subject: [PATCH 03/30] Add perf-changelog.yaml entry for dsv4-fp4-mi355x-sglang-disagg-agentic-hicache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves review comment: the image bump (v0.5.14 -> v0.5.15.post1), re-enabled TP8/EP8/DPA hicache arm, and HiCache sizing/NCCL/router fixes need a changelog entry to trigger the benchmark sweep on merge. 中文：为 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache 添加 perf-changelog.yaml 条目。解决评审意见：镶像升级（v0.5.14 -> v0.5.15.post1）、重新启用的 TP8/EP8/DPA hicache 分支，以及 HiCache 容量/NCCL/路由策略修复，需要一条 changelog 条目才能在合并时触发基准测试扫描。 Co-authored-by: Cursor --- perf-changelog.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 09108c33ea..e178816066 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5159,3 +5159,14 @@ - "Bring-up validated in run 30326393603: all 12 configs green, zero ServerDisconnectedError after the keep-alive fix. GPU KV resolves to 42.23 GiB / 3,249,215 tokens, i.e. ~3.1 max-length requests, against MAX_NUM_SEQS = 2*CONC." - "Measured behaviour: below conc 8 the GPU-resident and DRAM arms are within run-to-run noise (1-5%). At conc 16 and 24 the GPU-resident arm thrashes -- prefix cache hit rate 2.7%, TTFT p50 86s and 191s, 49.6 and 54.9 output tok/s -- because the working set exceeds GPU KV and prefixes are recomputed. The DRAM arm holds TTFT p50 0.85s and 6.2s for 245.0 and 260.6 output tok/s (4-5x), with the CPU tier serving a 62% external prefix cache hit rate at conc 24. The high-conc GPU-resident points are retained deliberately as the honest baseline that makes the offload gain legible." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2371 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Bump image lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 -> v0.5.15.post1-rocm720-mi35x-20260719." + - "HiCache sizing: recipe now sets HICACHE_RATIO=4 and FORCE_HICACHE_RATIO=1 to explicitly opt into ratio-based sizing (DeepSeek-V4's hybrid HiCache pool rejects --hicache-size); job.slurm now forwards FORCE_HICACHE_RATIO into the container HiCache env file, where it was previously silently dropped." + - "Decode cuda-graph crash workaround: disable prefill cuda-graph for DeepSeek-V4-Pro-AgentX, drop --enable-two-batch-overlap from dp_flags, lower prefill mem_fraction_static 0.8->0.72 and no-dp max_running_requests 128->64; add TORCH_NCCL_BLOCKING_WAIT/NCCL_BLOCKING_WAIT=1 to avoid a ROCm 7.2 HIP-runtime watchdog race during decode cuda-graph capture." + - "Fix router policy env-var bug: server_sglang.sh read ROUTER_PREFILL_POLICY but recipes export PREFILL_ROUTER_POLICY, so the recipe override was silently ignored; also switch the agentic default prefill router policy cache_aware -> consistent_hashing and add --dp-aware (superseding the separate --decode-policy override)." + - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." + - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 From 58570ff821cde7042b4aac532dc6cfa41869f93c Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 08:08:13 +0000 Subject: [PATCH 04/30] remove disable_cuda_graph: false setting from decode Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/env.sh | 2 +- benchmarks/multi_node/amd_utils/models.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 2549e63af2..a4c557d860 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -293,7 +293,7 @@ else # event is queried during capture. CUDA graph stays fully enabled. export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}" export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" - export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" + # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" # ========================================================================= # DeepSeek-V4-Pro PD recipe overrides diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 317b51ce28..f9e06ac120 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -388,7 +388,6 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 decode: mem_fraction_static: 0.85 - disable_cuda_graph: false prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: From 7bf9be006d4401c7f217144a81b53a94f4dcc705 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 08:24:36 +0000 Subject: [PATCH 05/30] add env PREFILL_DP_ATTN and DECODE_DP_ATTN Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/server_sglang.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index f1124e5205..78c546e616 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -995,8 +995,8 @@ if [ "$NODE_RANK" -eq 0 ]; then ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ DISAGG IS_MULTINODE \ TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_ENABLE_DP PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_ENABLE_DP DECODE_HARDWARE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_ENABLE_DP PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_ENABLE_DP DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ From 48e703c1022380c057d0af743d8c8511c4ccbdbf Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 27 Jul 2026 06:45:37 +0000 Subject: [PATCH 06/30] fix(dsv4): forward PREFILL_EP/DECODE_EP/*_DP_ATTN into agentic server container job.slurm only forwarded the PREFILL_ENABLE_EP/PREFILL_ENABLE_DP launch booleans into the docker run env, not the raw PREFILL_EP/PREFILL_DP_ATTN (and decode counterparts) that process_agentic_result.py reads for agg_bmk.json's parallelism metadata. Without them the metadata silently defaulted to ep=1/dp_attention=false for every multinode agentic run -- including DEP8 (ep=8, dp-attn=true) arms -- which is why the dashboard mislabeled those points as "1xTP8+1xTP8" instead of "DEP8". Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/job.slurm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 3c86f6fa06..10bc126ff5 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -445,6 +445,11 @@ DOCKER_ENV_COMMON=( -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + # process_agentic_result.py reads these (distinct from the ENABLE_* launch + # booleans above) for agg_bmk.json's parallelism metadata; without them it + # defaulted to ep=1/dp_attention=false even for DEP8 arms. + -e PREFILL_EP=\${PREFILL_EP:-1} + -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN:-false} -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} @@ -466,6 +471,9 @@ DOCKER_ENV_COMMON=( -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP + # See PREFILL_EP/PREFILL_DP_ATTN comment above -- decode counterpart. + -e DECODE_EP=\${DECODE_EP:-1} + -e DECODE_DP_ATTN=\${DECODE_DP_ATTN:-false} -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE -e DRY_RUN=\${DRY_RUN:-0} From 5752932c828505e4c1b56fe5d58eb6227708fc74 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 27 Jul 2026 08:24:40 +0000 Subject: [PATCH 07/30] feat(agentic): add multi-node agentic (SWE-bench) eval dispatch support Splits multi-node agentic eval rows into their own matrix bucket (multinode_agentic_evals) and dispatches them via new sweep-multi-node- agentic-evals / test-sweep-multi-node-agentic-evals jobs, since SWE-bench doesn't support batched concurrencies the way lm-eval does. server_sglang.sh gains the IS_AGENTIC_RUN branch that runs SWE-bench eval-only (mirroring the single-node agentic recipe) and stages its artifacts for NFS extraction by the runner. Co-authored-by: Cursor --- .../workflows/benchmark-multinode-tmpl.yml | 21 +++++ .github/workflows/e2e-tests.yml | 70 ++++++++++++++- .github/workflows/run-sweep.yml | 76 ++++++++++++++++- .../multi_node/amd_utils/server_sglang.sh | 64 +++++++++++++- utils/evals/EVALS.md | 34 ++++++-- utils/matrix_logic/generate_sweep_configs.py | 64 ++++++++++++-- .../test_generate_sweep_configs.py | 85 +++++++++++++++++++ utils/matrix_logic/test_validation.py | 69 +++++++++++++++ utils/matrix_logic/validation.py | 13 +++ utils/process_changelog.py | 15 +++- utils/recover_failed_ingest.py | 3 +- utils/test_process_changelog.py | 63 ++++++++++++++ 12 files changed, 553 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 23f766c388..4ecf29b9b3 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -138,6 +138,16 @@ on: type: string required: false default: "" + eval-limit: + description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" + required: false + type: string + default: "" + swebench-gen-mode: + description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." + required: false + type: string + default: "" # Agentic-coding inputs. Fixed-seq-len jobs leave these empty. scenario-type: description: "Scenario type (fixed-seq-len or agentic-coding)" @@ -213,6 +223,12 @@ env: RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} EVAL_CONC: ${{ inputs.eval-conc }} + EVAL_LIMIT: ${{ inputs.eval-limit }} + SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} + # GPU/multi-node runners lack Docker for SWE-bench scoring. + SWEBENCH_USE_MODAL: 'true' + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} @@ -422,6 +438,10 @@ jobs: meta_env.json results*.json sample*.jsonl + agent_preds.json + predictions.jsonl + swebench_report_*.json + *.traj* if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }} - name: Verify eval scores @@ -439,6 +459,7 @@ jobs: rm -f meta_env.json || true rm -f results*.json || true rm -f sample*.jsonl || true + rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true - name: Slurm cleanup (post-run) if: always() diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 0420cf73dc..84a88218f8 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -82,6 +82,7 @@ jobs: agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }} multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} + multi-node-agentic-eval-config: ${{ steps.get-jobs.outputs.multi-node-agentic-eval-config }} steps: - name: Checkout code (ref) if: ${{ inputs.ref && inputs.ref != '' }} @@ -124,14 +125,16 @@ jobs: } AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic) AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval) - MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) + MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic) + MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) - MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) + MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT echo "agentic-eval-config=$AGENTIC_EVAL" >> $GITHUB_OUTPUT echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT + echo "multi-node-agentic-eval-config=$MULTI_AGENTIC_EVAL" >> $GITHUB_OUTPUT echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT echo "eval-config=$EVALS" >> $GITHUB_OUTPUT @@ -382,6 +385,65 @@ jobs: scenario-type: agentic-coding ref: ${{ inputs.ref }} + test-sweep-multi-node-agentic-evals: + needs: get-jobs + if: ${{ needs.get-jobs.outputs.multi-node-agentic-eval-config != '[]' }} + uses: ./.github/workflows/benchmark-multinode-tmpl.yml + name: multi-node agentic eval / + strategy: + fail-fast: false + matrix: + config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-eval-config) }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + isl: '0' + osl: '0' + max-model-len: '0' + runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} + conc-list: ${{ toJson(matrix.config.conc) }} + spec-decoding: ${{ matrix.config.spec-decoding }} + disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} + prefill-num-worker: ${{ matrix.config.prefill.num-worker }} + prefill-tp: ${{ matrix.config.prefill.tp }} + prefill-pp: ${{ matrix.config.prefill.pp }} + prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} + prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} + prefill-ep: ${{ matrix.config.prefill.ep }} + prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} + prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} + decode-num-worker: ${{ matrix.config.decode.num-worker }} + decode-tp: ${{ matrix.config.decode.tp }} + decode-pp: ${{ matrix.config.decode.pp }} + decode-dcp-size: ${{ matrix.config.decode.dcp-size }} + decode-pcp-size: ${{ matrix.config.decode.pcp-size }} + decode-ep: ${{ matrix.config.decode.ep }} + decode-dp-attn: ${{ matrix.config.decode.dp-attn }} + decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} + conc: ${{ matrix.config['eval-conc'] }} + kv-offloading: ${{ matrix.config.kv-offloading }} + kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} + kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} + duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} + run-eval: true + eval-only: true + eval-conc: ${{ matrix.config['eval-conc'] }} + eval-limit: ${{ inputs.eval-limit }} + swebench-gen-mode: ${{ inputs.swebench-gen-mode }} + scenario-type: agentic-coding + ref: ${{ inputs.ref }} + test-sweep-single-node: needs: get-jobs if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} @@ -467,8 +529,8 @@ jobs: result-prefix: "bmk" collect-evals: - needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] - if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} + needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals, test-sweep-multi-node-agentic-evals] + if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped' || needs.test-sweep-multi-node-agentic-evals.result != 'skipped') }} uses: ./.github/workflows/collect-evals.yml secrets: inherit diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 7c849cb57d..5063fbf627 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -846,6 +846,78 @@ jobs: eval-only: true eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} + # Multi-node agentic (SWE-bench) eval rows carry the agentic input shape, + # so they are dispatched with sweep-multi-node-agentic's inputs rather + # than sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, + # which agentic rows don't have). SWE-bench doesn't support batched + # concurrencies (unlike lm-eval), so eval-conc is always a single value, + # never the joined-list form sweep-multi-node-evals uses. + sweep-multi-node-agentic-evals: + needs: [setup, canary-select, canary-sweep] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.setup.outputs.reuse-enabled != 'true' && + (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && + toJson(fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals) != '[]' && + toJson(fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals) != 'null' + }} + uses: ./.github/workflows/benchmark-multinode-tmpl.yml + name: multi-node agentic eval / + strategy: + fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }} + matrix: + config: ${{ fromJson(needs.setup.outputs.search-space-config).multinode_agentic_evals }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + isl: '0' + osl: '0' + max-model-len: '0' + runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} + conc-list: ${{ toJson(matrix.config.conc) }} + spec-decoding: ${{ matrix.config.spec-decoding }} + disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} + prefill-num-worker: ${{ matrix.config.prefill.num-worker }} + prefill-tp: ${{ matrix.config.prefill.tp }} + prefill-pp: ${{ matrix.config.prefill.pp }} + prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} + prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} + prefill-ep: ${{ matrix.config.prefill.ep }} + prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} + prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} + decode-num-worker: ${{ matrix.config.decode.num-worker }} + decode-tp: ${{ matrix.config.decode.tp }} + decode-pp: ${{ matrix.config.decode.pp }} + decode-dcp-size: ${{ matrix.config.decode.dcp-size }} + decode-pcp-size: ${{ matrix.config.decode.pcp-size }} + decode-ep: ${{ matrix.config.decode.ep }} + decode-dp-attn: ${{ matrix.config.decode.dp-attn }} + decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} + conc: ${{ matrix.config['eval-conc'] }} + kv-offloading: ${{ matrix.config.kv-offloading }} + kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} + kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} + total-cpu-dram-gb: ${{ matrix.config['total-cpu-dram-gb'] }} + duration: ${{ matrix.config.duration }} + run-eval: true + eval-only: true + eval-conc: ${{ matrix.config['eval-conc'] }} + scenario-type: agentic-coding + collect-results: needs: [ @@ -876,8 +948,8 @@ jobs: result-prefix: "bmk" collect-evals: - needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup] - if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }} + needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, sweep-multi-node-agentic-evals, setup] + if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped' || needs.sweep-multi-node-agentic-evals.result != 'skipped') }} uses: ./.github/workflows/collect-evals.yml secrets: inherit diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 78c546e616..57cd69f21e 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1069,7 +1069,11 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { # Run evaluation if requested (before killing router) if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." + if [[ "$IS_AGENTIC_RUN" == "1" ]]; then + echo "Running SWE-bench agentic evaluation on Node 0..." + else + echo "Running lm-eval evaluation on Node 0..." + fi # Health check: verify the router is still serving before running eval. # The throughput benchmark may have crashed/exhausted decode workers. @@ -1085,6 +1089,64 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { if [[ "$EVAL_HEALTH_OK" != "true" ]]; then echo "WARNING: Router health check failed after 3 attempts. Skipping eval." + elif [[ "$IS_AGENTIC_RUN" == "1" ]]; then + # Multi-node agentic (SWE-bench) eval-only. This mirrors the + # single-node agentic eval-only recipe + # (benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh): + # run_eval() auto-selects the swebench framework via + # IS_AGENTIC/SCENARIO_TYPE and stages meta_env.json/results*.json + # itself (EVAL_ONLY=true + agentic), so there's no separate + # append_lm_eval_summary call here like the lm-eval path below. + pushd /workspace + source /workspace/benchmarks/benchmark_lib.sh + + # Bridge the few metadata field names the workflow spells + # differently from what append_lm_eval_summary expects + # (PREFILL_DP_ATTN vs PREFILL_DP_ATTENTION), and fill in the + # single-engine TP/EP/DP fields from the prefill side, since + # disaggregated topologies have no single "TP" of their own. + export TP="${PREFILL_TP:-1}" + export EP_SIZE="${PREFILL_EP:-1}" + export DP_ATTENTION="${PREFILL_DP_ATTN:-false}" + export PREFILL_DP_ATTENTION="${PREFILL_DP_ATTN:-false}" + export DECODE_DP_ATTENTION="${DECODE_DP_ATTN:-false}" + + # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list + if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" + else + export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) + fi + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: run_eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS})" + else + run_eval --port 30000 + eval_rc=$? + + if [[ $eval_rc -ne 0 ]]; then + echo "ERROR: run_eval exited rc=$eval_rc; skipping eval artifact staging" >&2 + EVAL_FAILED=1 + else + # Files (meta_env.json, results*.json, sample*.jsonl, ...) are now in /workspace + EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" + mkdir -p "$EVAL_COPY_DIR" + for f in meta_env.json; do + [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" + done + # Use find for glob patterns to avoid "no match" errors + find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name 'agent_preds.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name 'predictions.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name 'swebench_report_*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name '*.traj*' -exec cp -f {} "$EVAL_COPY_DIR/" \; + + echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" + fi + fi + + popd else # Must run from repo root so utils/evals/${task}.yaml resolves pushd /workspace diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 5d07f887f6..9267274bc8 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -11,6 +11,11 @@ from throughput; selection lives in `mark_eval_entries()` in runner, framework, precision, TP, and decoding configuration. - **Multi-node:** 8k1k only; one job per parallelism topology at its highest eligible concurrency. Rows differing only by concurrency share a topology. +- **Agentic (SWE-bench), single-node:** highest-conc entry per (model, + runner, framework, precision) group. +- **Agentic (SWE-bench), multi-node:** same policy as multi-node fixed-seq-len + above (highest eligible conc per parallelism topology), since SWE-bench + doesn't support batched concurrencies the way lm-eval does. Generator eval modes: @@ -84,9 +89,16 @@ In eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MO ### Multi-node Multi-node evals support two hardware paths: -**MI355X (AMD)** — `benchmarks/multi_node/amd_utils/server.sh` -- Skips `bench.sh` when `EVAL_ONLY=true` -- Runs lm-eval via `run_eval` against the router on port 30000 +**MI355X (AMD)** — `benchmarks/multi_node/amd_utils/server_sglang.sh` +- Skips throughput when `EVAL_ONLY=true` +- Fixed-seq-len: runs lm-eval via `run_eval --framework lm-eval` against the router on port 30000 +- Agentic-coding (disaggregated, `IS_AGENTIC=1`): runs SWE-bench via `run_eval --port 30000` (no + `--framework` override, same auto-selection as single-node agentic eval-only). Since there's no + single "TP" for a disaggregated topology, and the workflow spells a couple of metadata fields + differently (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects + (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before calling + `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` (same + `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call. - Concurrency uses workflow-provided `EVAL_CONC` when set, otherwise falls back to max of `BENCH_MAX_CONCURRENCY` (x-separated values) - Eval artifacts copied to `/run_logs/slurm_job-*/eval_results/` - `runners/launch_mi355x-amds.sh` skips benchmark result collection when `EVAL_ONLY=true` and uses `find` to locate eval results @@ -102,11 +114,17 @@ Multi-node evals support two hardware paths: For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it contains multiple values, `run_eval` runs those concurrency points sequentially against the same live engine, stages each result with a `_concN` filename suffix, and records expected/completed/failed points in `meta_env.json`. ### Workflow structure -- `e2e-tests.yml`: `test-sweep-evals` (single-node) and `test-sweep-multi-node-evals` (multi-node) -- `run-sweep.yml`: `sweep-evals` (single-node) and `sweep-multi-node-evals` (multi-node) -- Both use their respective benchmark templates with `eval-only: true`, `run-eval: true` -- `collect-evals` depends on both eval jobs; `collect-results` only runs when benchmark jobs ran -- `process_changelog.py` splits eval results into `evals` (single-node) and `multinode_evals` +- `e2e-tests.yml`: `test-sweep-evals` (single-node fixed-seq-len), `test-sweep-multi-node-evals` + (multi-node fixed-seq-len), `test-sweep-agentic-evals` (single-node agentic), and + `test-sweep-multi-node-agentic-evals` (multi-node agentic) +- `run-sweep.yml`: `sweep-evals`, `sweep-multi-node-evals`, `sweep-agentic-evals`, and + `sweep-multi-node-agentic-evals` (same four-way split) +- All four use their respective benchmark templates (`benchmark-tmpl.yml` for single-node, + `benchmark-multinode-tmpl.yml` for multi-node) with `eval-only: true`, `run-eval: true` +- `collect-evals` depends on all four eval jobs; `collect-results` only runs when benchmark jobs ran +- `process_changelog.py` splits eval results by node count and scenario type into `evals` + (single-node fixed-seq-len), `agentic_evals` (single-node agentic), `multinode_evals` + (multi-node fixed-seq-len), and `multinode_agentic_evals` (multi-node agentic) ### Result collection diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 7df1a75b64..0a1a0e88a8 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -220,13 +220,22 @@ def _freeze_matrix_value(value): def _multinode_parallelism_key(entry: dict) -> tuple: - """Identify a multi-node config independently of eval/concurrency fields.""" + """Identify a multi-node config independently of eval/concurrency fields. + + exp-name is derived from (and ignored alongside) conc: fixed-seq-len + exp-names never embed conc, but agentic exp-names do (each concurrency + gets its own single-conc allocation, per + MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION), so entries for the + same topology at different concurrencies would otherwise falsely land in + different groups. + """ ignored_fields = { Fields.CONC.value, Fields.RUN_EVAL.value, Fields.EVAL_ONLY.value, Fields.EVAL_CONC.value, Fields.EVAL_ALL_CONCS.value, + Fields.EXP_NAME.value, } return tuple(sorted( (key, _freeze_matrix_value(value)) @@ -248,6 +257,11 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) - Mark the entry containing its highest eligible concurrency - Set eval-conc to that highest eligible concurrency - Agentic evals are opt-in to preserve default throughput coverage. + - Single-node: mark the highest-conc entry per (model, runner, + framework, precision) group. + - Multi-node: same policy as the fixed-seq-len multi-node case above + (highest eligible conc per distinct parallelism config, via + eval-conc), since SWE-bench doesn't support batched concurrencies. """ from collections import defaultdict @@ -313,11 +327,19 @@ def _eligible_eval_concs(entry): # Default sweeps preserve every agentic throughput result. if include_agentic: ag_sn_groups = defaultdict(list) + # Multi-node agentic: same "highest eligible conc per distinct + # parallelism config" policy as the fixed-seq-len mn_groups above. + # SWE-bench doesn't support batched concurrencies (unlike lm-eval), + # so exactly one conc is picked per group, never the full list. + ag_mn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding': continue - # Multi-node agentic eval is unsupported. if Fields.PREFILL.value in entry: + eval_concs = _eligible_eval_concs(entry) + if not eval_concs: + continue + ag_mn_groups[_multinode_parallelism_key(entry)].append((i, eval_concs[-1])) continue conc = entry[Fields.CONC.value] conc_val = max(conc) if isinstance(conc, list) else conc @@ -330,6 +352,10 @@ def _eligible_eval_concs(entry): ag_sn_groups[key].append((i, conc_val)) for entries in ag_sn_groups.values(): eval_indices.add(max(entries, key=lambda item: item[1])[0]) + for entries in ag_mn_groups.values(): + best_idx, best_eval_conc = max(entries, key=lambda item: item[1]) + eval_indices.add(best_idx) + mn_eval_conc[best_idx] = best_eval_conc for i, entry in enumerate(matrix_values): entry[Fields.RUN_EVAL.value] = i in eval_indices @@ -345,12 +371,18 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: Evals only run at 8k1k (matching mark_eval_entries), so entries at other sequence lengths (e.g. 1k1k) are passed through untouched rather than expanded into eval rows. - Single-node agentic entries use SWE-bench; multi-node eval is unsupported. - Multi-node rows with the same engine topology are merged into one eval row - whose full concurrency list is run sequentially against the same engine. + Single-node and multi-node agentic entries use SWE-bench, which doesn't + support batched concurrencies (unlike lm-eval): multi-node agentic rows + with the same topology are merged (to recombine any chunking split), but + only the highest resulting conc is marked for eval via eval-conc, not the + full list. + Multi-node fixed-seq-len rows with the same engine topology are merged + into one eval row whose full concurrency list is run sequentially + against the same engine. """ expanded_entries: list[dict] = [] multinode_indices: dict[tuple, int] = {} + multinode_agentic_indices: dict[tuple, int] = {} target_isl, target_osl = seq_len_stoi["8k1k"] @@ -358,7 +390,27 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: if entry.get(Fields.SCENARIO_TYPE.value) == 'agentic-coding': if Fields.PREFILL.value not in entry: entry[Fields.RUN_EVAL.value] = True - expanded_entries.append(entry) + expanded_entries.append(entry) + continue + + conc = entry[Fields.CONC.value] + conc_values = conc if isinstance(conc, list) else [conc] + parallelism_key = _multinode_parallelism_key(entry) + if parallelism_key in multinode_agentic_indices: + existing = expanded_entries[multinode_agentic_indices[parallelism_key]] + merged_conc = sorted(set(existing[Fields.CONC.value] + conc_values)) + existing[Fields.CONC.value] = merged_conc + existing[Fields.EVAL_CONC.value] = max(merged_conc) + continue + + eval_entry = { + **entry, + Fields.CONC.value: sorted(set(conc_values)), + Fields.RUN_EVAL.value: True, + Fields.EVAL_CONC.value: max(conc_values), + } + multinode_agentic_indices[parallelism_key] = len(expanded_entries) + expanded_entries.append(eval_entry) continue # Only 8k1k is eligible for evals; leave other sequence lengths as-is diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 1ad4e3c922..f5279ce2d5 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -233,6 +233,66 @@ def test_marks_agentic_entry_for_swebench(self): assert len(marked) == 1 assert marked[0]["conc"] == 64 + def test_marks_multinode_agentic_entry_at_highest_eligible_conc(self): + """Multi-node agentic (SWE-bench) eval selection mirrors the + fixed-seq-len multi-node policy: one eval row per parallelism + topology, at its highest eligible (>= MIN_EVAL_CONC) concurrency. + + Each concurrency is its own matrix entry (chunk size 1) whose + exp-name embeds that concurrency, unlike fixed-seq-len multi-node + rows where exp-name never varies with conc — the grouping key must + still treat these as the same topology. + """ + common = { + "scenario-type": "agentic-coding", + "model": "m", "runner": "b300", "framework": "sglang-disagg", + "precision": "fp4", "spec-decoding": "none", "disagg": True, + "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + } + matrix_values = [ + {**common, "conc": [8], "exp-name": "p1x8_d1x8_conc8"}, + {**common, "conc": [16], "exp-name": "p1x8_d1x8_conc16"}, + {**common, "conc": [32], "exp-name": "p1x8_d1x8_conc32"}, + ] + + result = mark_eval_entries(matrix_values, include_agentic=True) + + marked = [e for e in result if e.get("run-eval")] + assert len(marked) == 1 + assert marked[0]["conc"] == [32] + assert marked[0]["eval-conc"] == 32 + + def test_multinode_agentic_groups_are_independent_per_topology(self): + """Two distinct multi-node agentic topologies (e.g. differing by + prefill EP/DP) must each get their own eval row.""" + base = { + "scenario-type": "agentic-coding", + "model": "m", "runner": "b300", "framework": "sglang-disagg", + "precision": "fp4", "spec-decoding": "none", "disagg": True, + } + topology_a = { + "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + } + topology_b = { + "prefill": {"num-worker": 1, "tp": 8, "ep": 8, "dp-attn": True}, + "decode": {"num-worker": 1, "tp": 8, "ep": 8, "dp-attn": True}, + } + matrix_values = [ + {**base, **topology_a, "conc": [16], "exp-name": "a_conc16"}, + {**base, **topology_a, "conc": [32], "exp-name": "a_conc32"}, + {**base, **topology_b, "conc": [64], "exp-name": "b_conc64"}, + {**base, **topology_b, "conc": [96], "exp-name": "b_conc96"}, + ] + + result = mark_eval_entries(matrix_values, include_agentic=True) + + marked = {e["exp-name"]: e for e in result if e.get("run-eval")} + assert set(marked) == {"a_conc32", "b_conc96"} + assert marked["a_conc32"]["eval-conc"] == 32 + assert marked["b_conc96"]["eval-conc"] == 96 + def test_default_mode_does_not_mark_agentic(self): matrix_values = [ { @@ -677,6 +737,31 @@ def test_marks_agentic_entries_for_swebench(self): assert result[0]['run-eval'] is True assert 'eval-conc' not in result[0] + + def test_marks_multinode_agentic_entries_for_swebench(self): + """Unlike fixed-seq-len multi-node (which batches every concurrency + into one lm-eval row via eval-all-concs), multi-node agentic rows for + the same topology are merged but only their highest conc is marked + via eval-conc, since SWE-bench doesn't support batched concurrencies.""" + common = { + 'scenario-type': 'agentic-coding', + 'model': 'm', 'runner': 'r', 'framework': 'sglang-disagg', + 'precision': 'fp4', 'spec-decoding': 'none', 'disagg': True, + 'prefill': {'num-worker': 1, 'tp': 8, 'ep': 1, 'dp-attn': False}, + 'decode': {'num-worker': 1, 'tp': 8, 'ep': 1, 'dp-attn': False}, + } + entries = [ + {**common, 'conc': [2], 'exp-name': 'p1x8_d1x8_conc2'}, + {**common, 'conc': [16], 'exp-name': 'p1x8_d1x8_conc16'}, + {**common, 'conc': [32], 'exp-name': 'p1x8_d1x8_conc32'}, + ] + + result = mark_all_eval_entries(entries) + + assert len(result) == 1 + assert result[0]['run-eval'] is True + assert result[0]['conc'] == [2, 16, 32] + assert result[0]['eval-conc'] == 32 assert 'eval-all-concs' not in result[0] diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 3d25d03eef..ba44514339 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -8,6 +8,7 @@ SingleNodeMatrixEntry, SingleNodeAgenticMatrixEntry, MultiNodeMatrixEntry, + MultiNodeAgenticMatrixEntry, WorkerConfig, SingleNodeSearchSpaceEntry, AgenticCodingConfig, @@ -20,6 +21,7 @@ ChangelogEntry, ChangelogMatrixEntry, validate_matrix_entry, + validate_agentic_matrix_entry, validate_master_config, validate_runner_config, load_config_files, @@ -1415,6 +1417,20 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type): "scenario-type": "agentic-coding", "run-eval": True, "eval-only": True, } +MULTINODE_AGENTIC_EVAL_ROW = { + "image": "lmsysorg/sglang-rocm:v0.5.15", "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", "precision": "fp4", "framework": "sglang-disagg", + "spec-decoding": "none", "runner": "cluster:mi355x-amds", + "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "conc": [32], "kv-offloading": "dram", + "kv-offload-backend": {"name": "hicache"}, "kv-p2p-transfer": "mori", + "total-cpu-dram-gb": 2399, "duration": 3600, + "exp-name": "dsv4_p1x8_d1x8_conc32_kvdram-hicache", "disagg": True, + "scenario-type": "agentic-coding", "run-eval": True, "eval-only": True, + "eval-conc": 32, +} + CHANGELOG_METADATA = { "base_ref": "base", "head_ref": "head", "entries": [{ @@ -1443,6 +1459,59 @@ def test_agentic_eval_rows_live_in_agentic_evals_only(self): "changelog_metadata": CHANGELOG_METADATA, }) + def test_multinode_agentic_eval_rows_live_in_multinode_agentic_evals_only(self): + """multinode_evals is dispatched with fixed-seq-len inputs (isl/osl/ + max-model-len), so multi-node agentic rows must only validate in + multinode_agentic_evals.""" + entry = ChangelogMatrixEntry.model_validate({ + "multinode_agentic_evals": [MULTINODE_AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, + }) + assert entry.multinode_agentic_evals[0].run_eval is True + assert entry.multinode_agentic_evals[0].eval_only is True + assert entry.multinode_agentic_evals[0].eval_conc == 32 + + with pytest.raises(ValueError): + ChangelogMatrixEntry.model_validate({ + "multinode_evals": [MULTINODE_AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, + }) + + +class TestMultiNodeAgenticMatrixEntry: + """Tests for multi-node agentic (SWE-bench) matrix entry validation.""" + + def test_throughput_row_omits_eval_fields(self): + entry = MultiNodeAgenticMatrixEntry(**{ + k: v for k, v in MULTINODE_AGENTIC_EVAL_ROW.items() + if k not in ("run-eval", "eval-only", "eval-conc") + }) + assert entry.run_eval is None + assert entry.eval_only is None + assert entry.eval_conc is None + + def test_eval_row_carries_run_eval_eval_only_and_eval_conc(self): + entry = MultiNodeAgenticMatrixEntry(**MULTINODE_AGENTIC_EVAL_ROW) + assert entry.run_eval is True + assert entry.eval_only is True + assert entry.eval_conc == 32 + + def test_validate_agentic_matrix_entry_dispatches_on_prefill_key(self): + """The dispatcher in validate_agentic_matrix_entry() picks + MultiNodeAgenticMatrixEntry vs SingleNodeAgenticMatrixEntry based on + whether the entry has a `prefill` key.""" + validate_agentic_matrix_entry(dict(MULTINODE_AGENTIC_EVAL_ROW)) + + without_prefill = { + k: v for k, v in MULTINODE_AGENTIC_EVAL_ROW.items() + if k not in ("prefill", "decode") + } + with pytest.raises(ValueError): + # Missing single-node-required fields (tp, pp, ...) and conc is a + # list rather than a scalar, so this must fail as a single-node + # agentic entry rather than silently validating. + validate_agentic_matrix_entry(without_prefill) + # ============================================================================= # Test load_config_files diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..1b0efc8a19 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -334,6 +334,13 @@ class MultiNodeAgenticMatrixEntry(BaseModel): exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + # Agentic eval rows (SWE-bench) carry run-eval/eval-only/eval-conc; + # benchmark rows omit them, and exclude_none keeps them out of dumped + # throughput output. SWE-bench doesn't support batched concurrencies + # (unlike lm-eval), so there is no eval-all-concs field here. + run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value) + eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value) + eval_conc: Optional[int] = Field(default=None, alias=Fields.EVAL_CONC.value) @model_validator(mode='after') def validate_worker_hardware_pair(self): @@ -903,6 +910,12 @@ class ChangelogMatrixEntry(BaseModel): agentic_evals: list[SingleNodeAgenticMatrixEntry] = Field( default_factory=list) multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list) + # Multi-node agentic (SWE-bench) eval rows, split out of multinode_evals + # the same way agentic_evals is split out of evals: they carry the + # agentic input shape (scenario-type, kv-offloading, ...) rather than + # the fixed-seq-len shape (isl/osl/max-model-len) multinode_evals rows do. + multinode_agentic_evals: list[MultiNodeAgenticMatrixEntry] = Field( + default_factory=list) changelog_metadata: ChangelogMetadata diff --git a/utils/process_changelog.py b/utils/process_changelog.py index c4b226ada0..b885b9b5f7 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -139,6 +139,7 @@ def main(): "evals": [], "agentic_evals": [], "multinode_evals": [], + "multinode_agentic_evals": [], "changelog_metadata": { "base_ref": args.base_ref, "head_ref": args.head_ref, @@ -274,8 +275,11 @@ def main(): # Agentic eval rows go to their own bucket so run-sweep.yml can dispatch # them with agentic inputs (scenario-type, kv-offloading, ...) instead of - # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have. + # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have. Same + # split applies on the multi-node side (multinode_evals vs + # multinode_agentic_evals). single_node_evals = [e for e in all_eval_results if e.get("prefill") is None] + multi_node_evals = [e for e in all_eval_results if e.get("prefill") is not None] final_results["evals"] = [ e for e in single_node_evals if e.get("scenario-type") != "agentic-coding" @@ -284,7 +288,14 @@ def main(): e for e in single_node_evals if e.get("scenario-type") == "agentic-coding" ] - final_results["multinode_evals"] = [e for e in all_eval_results if e.get("prefill") is not None] + final_results["multinode_evals"] = [ + e for e in multi_node_evals + if e.get("scenario-type") != "agentic-coding" + ] + final_results["multinode_agentic_evals"] = [ + e for e in multi_node_evals + if e.get("scenario-type") == "agentic-coding" + ] # Validate final results structure validated = ChangelogMatrixEntry.model_validate(final_results) diff --git a/utils/recover_failed_ingest.py b/utils/recover_failed_ingest.py index f6bb686098..892fcbfffa 100644 --- a/utils/recover_failed_ingest.py +++ b/utils/recover_failed_ingest.py @@ -493,7 +493,8 @@ def build_config( + len(config.get("multi_node", {}).get("agentic", []) or []), "eval_jobs": len(config.get("evals", []) or []) + len(config.get("agentic_evals", []) or []) - + len(config.get("multinode_evals", []) or []), + + len(config.get("multinode_evals", []) or []) + + len(config.get("multinode_agentic_evals", []) or []), } diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 476bec0742..7f6316b615 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -639,3 +639,66 @@ def fake_run(command, **kwargs): assert [r["exp-name"] for r in output["evals"]] == ["fixed_eval"] assert [r["exp-name"] for r in output["agentic_evals"]] == ["agentic_eval"] assert output["multinode_evals"] == [] + + +def test_eval_rows_split_into_multinode_fixed_and_agentic_buckets( + monkeypatch, + capsys, +): + """Multi-node eval rows must split the same way single-node rows do: + fixed-seq-len rows in `multinode_evals`, agentic (SWE-bench) rows in + `multinode_agentic_evals`.""" + added_yaml = """ +- config-keys: + - test-config + description: + - Mixed multi-node fixed-seq-len and agentic eval selection + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1 +""" + common = { + "image": "lmsysorg/sglang-rocm:v0.5.15", "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", "precision": "fp4", "framework": "sglang-disagg", + "spec-decoding": "none", "runner": "cluster:mi355x-amds", + "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "disagg": True, "kv-p2p-transfer": "mori", + "run-eval": True, "eval-only": True, + } + multinode_fixed_eval_row = { + **common, "isl": 8192, "osl": 1024, "max-model-len": 10240, + "conc": [64], "eval-conc": 64, "exp-name": "multinode_fixed_eval", + } + multinode_agentic_eval_row = { + **common, "kv-offloading": "dram", + "kv-offload-backend": {"name": "hicache"}, + "total-cpu-dram-gb": 2399, "duration": 3600, + "scenario-type": "agentic-coding", + "conc": [32], "eval-conc": 32, "exp-name": "multinode_agentic_eval", + } + + monkeypatch.setattr( + process_changelog, "get_added_lines", lambda *_: added_yaml) + monkeypatch.setattr( + process_changelog, "load_config_files", lambda _: {"test-config": {}}) + + def fake_run(command, **kwargs): + is_evals = "--evals-only" in command + rows = ( + [multinode_fixed_eval_row, multinode_agentic_eval_row] + if is_evals else [] + ) + return SimpleNamespace(stdout=json.dumps(rows)) + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(sys, "argv", [ + "process_changelog.py", "--base-ref", "base", "--head-ref", "head", + "--changelog-file", "perf-changelog.yaml", + ]) + + process_changelog.main() + + output = json.loads(capsys.readouterr().out) + assert [r["exp-name"] for r in output["multinode_evals"]] == ["multinode_fixed_eval"] + assert [r["exp-name"] for r in output["multinode_agentic_evals"]] == ["multinode_agentic_eval"] + assert output["evals"] == [] + assert output["agentic_evals"] == [] From b1fd7ea75905f9cf3a54df1516ee2d76939985e3 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 27 Jul 2026 09:41:32 +0000 Subject: [PATCH 08/30] Mark dsv4-fp4-mi355x-sglang-disagg-agentic-hicache changelog entry evals-only Suppresses throughput benchmarking for this config-key and only runs the multi-node agentic (SWE-bench) eval at TP8/conc32 and DEP8/conc128. Co-authored-by: Cursor --- perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e178816066..1960a91149 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5170,3 +5170,4 @@ - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 + evals-only: true From 79ee7b6fc1932a4e38ddf57eda13f0df4232f84a Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 03:53:05 +0000 Subject: [PATCH 09/30] Fix multinode agentic eval-only CI: forward Modal creds and stage eval artifacts reliably. Modal/HF tokens were set in GitHub Actions but never reached the Slurm container, so SWE-bench sandboxes failed with missing credentials. Also use sudo when copying root-owned eval outputs so verify_scores sees fresh meta_env.json instead of stale workspace files. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/job.slurm | 10 ++++++++ benchmarks/multi_node/amd_utils/submit.sh | 4 +++ runners/launch_mi355x-amds.sh | 30 +++++++++++------------ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 10bc126ff5..c4731b9ab7 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -330,6 +330,10 @@ export RUNNER_TYPE="${RUNNER_TYPE:-}" export RESULT_FILENAME="${RESULT_FILENAME:-}" export SPEC_DECODING="${SPEC_DECODING:-}" export IS_MULTINODE="${IS_MULTINODE:-false}" +export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" +export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" +export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" +export HF_TOKEN="${HF_TOKEN:-}" SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" @@ -477,6 +481,12 @@ DOCKER_ENV_COMMON=( -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE -e DRY_RUN=\${DRY_RUN:-0} + # SWE-bench agentic eval runs inside this container and needs Modal/HF + # credentials to launch sandboxes and download datasets. + -e SWEBENCH_USE_MODAL=\${SWEBENCH_USE_MODAL:-false} + -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} + -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} + -e HF_TOKEN=\${HF_TOKEN:-} ) # Engine-specific env vars diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index f125eb0955..cb0c162138 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -145,6 +145,10 @@ export RUNNER_TYPE="${RUNNER_TYPE:-}" export RESULT_FILENAME="${RESULT_FILENAME:-}" export SPEC_DECODING="${SPEC_DECODING:-}" export IS_MULTINODE="${IS_MULTINODE:-false}" +export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" +export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" +export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" +export HF_TOKEN="${HF_TOKEN:-}" # Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 75a67c1c58..ab838d11f3 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -180,8 +180,18 @@ PY shopt -s nullglob for eval_file in "$EVAL_DIR"/*; do [ -f "$eval_file" ] || continue - cp "$eval_file" "$GITHUB_WORKSPACE/" - echo "Copied eval artifact: $(basename "$eval_file")" + eval_dest="$GITHUB_WORKSPACE/$(basename "$eval_file")" + rm -f "$eval_dest" + # Eval artifacts are created as root inside the container; sudo + # is required to overwrite any stale root-owned files in the + # workspace from prior runs on this runner. + if sudo cp "$eval_file" "$eval_dest"; then + sudo chown "$(id -u):$(id -g)" "$eval_dest" 2>/dev/null || true + echo "Copied eval artifact: $(basename "$eval_file")" + else + echo "ERROR: failed to copy eval artifact: $(basename "$eval_file")" >&2 + exit 1 + fi done shopt -u nullglob else @@ -210,20 +220,8 @@ PY echo "Staging agentic raw artifacts from $AGENTIC_SRC" mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" - # The source artifacts are created inside the container as root - # (--container-remap-root), so depending on how the runner - # invokes this script the copies can end up root-owned and/or - # read-only (aiperf/server_sglang make some dirs mode 0555). If - # the staged tree isn't owned+writable by the runner user, the - # next checkout's `git clean` fails with - # EACCES: permission denied, rmdir '.../LOGS/agentic'. - # chown to the invoking user (the same one that runs git clean) - # via sudo (already passwordless here for rm -rf). The follow-up - # chmod uses a+rwX (not just u+rwX): the *next* job against this - # same $GITHUB_WORKSPACE may be picked up by a different runner - # process running as a different OS user, which would otherwise - # fall outside the owner bits and still fail the same - # `git clean` with EACCES despite the chown above. + # Container artifacts arrive root-owned; chown/chmod so git clean + # and later jobs (possibly a different runner user) can remove LOGS/. sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" From 546b3428b3326b60c0c43c64da1d400f8c44103b Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 04:21:18 +0000 Subject: [PATCH 10/30] Harden multinode agentic Modal credential setup. Ensure ~/.modal.toml is written for agentic runs even when SWEBENCH_USE_MODAL is unset, and forward SCENARIO_TYPE/EVAL_LIMIT into the Slurm container alongside the Modal/HF tokens. Co-authored-by: Cursor --- benchmarks/benchmark_lib.sh | 9 +++++++-- benchmarks/multi_node/amd_utils/job.slurm | 4 ++++ benchmarks/multi_node/amd_utils/submit.sh | 2 ++ perf-changelog.yaml | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index c5e6fb2c86..fb329b922c 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1136,7 +1136,12 @@ _patch_swebench_scoring() { # SWE-bench requires ~/.modal.toml despite env credentials. _ensure_modal_credentials() { - if [ "${SWEBENCH_USE_MODAL:-false}" != "true" ]; then return 0; fi + # Agentic generation uses swerex_modal sandboxes even when scoring is local. + if [ "${SWEBENCH_USE_MODAL:-false}" != "true" ] \ + && [ "${IS_AGENTIC:-0}" != "1" ] \ + && [ "${SCENARIO_TYPE:-}" != "agentic-coding" ]; then + return 0 + fi # CI secrets may include whitespace or quotes. if [ -n "${MODAL_TOKEN_ID:-}" ]; then MODAL_TOKEN_ID=$(printf %s "$MODAL_TOKEN_ID" | tr -d "[:space:]\"'") @@ -1159,7 +1164,7 @@ _ensure_modal_credentials() { chmod 600 "$HOME/.modal.toml" echo "[swebench] wrote ~/.modal.toml from MODAL_TOKEN_ID/MODAL_TOKEN_SECRET env" else - echo "WARN: SWEBENCH_USE_MODAL=true but no ~/.modal.toml and no MODAL_TOKEN_ID/MODAL_TOKEN_SECRET env; Modal scoring will fail credential validation" >&2 + echo "WARN: Modal credentials required but no ~/.modal.toml and no MODAL_TOKEN_ID/MODAL_TOKEN_SECRET env; Modal sandboxes will fail authentication" >&2 fi } diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index c4731b9ab7..4ca4be3785 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -334,6 +334,8 @@ export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" export HF_TOKEN="${HF_TOKEN:-}" +export SCENARIO_TYPE="${SCENARIO_TYPE:-}" +export EVAL_LIMIT="${EVAL_LIMIT:-}" SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" @@ -487,6 +489,8 @@ DOCKER_ENV_COMMON=( -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} -e HF_TOKEN=\${HF_TOKEN:-} + -e SCENARIO_TYPE=\${SCENARIO_TYPE:-} + -e \"EVAL_LIMIT=\${EVAL_LIMIT:-}\" ) # Engine-specific env vars diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index cb0c162138..1493cc7f7f 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -149,6 +149,8 @@ export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" export HF_TOKEN="${HF_TOKEN:-}" +export SCENARIO_TYPE="${SCENARIO_TYPE:-}" +export EVAL_LIMIT="${EVAL_LIMIT:-}" # Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1960a91149..e178816066 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5170,4 +5170,3 @@ - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 - evals-only: true From 0251f657dea3b2bd39f7ac8adcd95870d94fc15e Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 09:02:43 +0000 Subject: [PATCH 11/30] Switch multinode agentic eval to GSM8K lm-eval with correct metadata. Route all RUN_EVAL paths through lm-eval instead of SWE-bench, export CONC and disagg topology fields before run_eval, and avoid duplicate append_lm_eval_summary for agentic eval-only. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 127 +++++------------- 1 file changed, 34 insertions(+), 93 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 57cd69f21e..edb5a96158 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1069,11 +1069,7 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { # Run evaluation if requested (before killing router) if [[ "${RUN_EVAL:-false}" == "true" ]]; then - if [[ "$IS_AGENTIC_RUN" == "1" ]]; then - echo "Running SWE-bench agentic evaluation on Node 0..." - else - echo "Running lm-eval evaluation on Node 0..." - fi + echo "Running lm-eval (GSM8K) evaluation on Node 0..." # Health check: verify the router is still serving before running eval. # The throughput benchmark may have crashed/exhausted decode workers. @@ -1089,87 +1085,51 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { if [[ "$EVAL_HEALTH_OK" != "true" ]]; then echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - elif [[ "$IS_AGENTIC_RUN" == "1" ]]; then - # Multi-node agentic (SWE-bench) eval-only. This mirrors the - # single-node agentic eval-only recipe - # (benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh): - # run_eval() auto-selects the swebench framework via - # IS_AGENTIC/SCENARIO_TYPE and stages meta_env.json/results*.json - # itself (EVAL_ONLY=true + agentic), so there's no separate - # append_lm_eval_summary call here like the lm-eval path below. - pushd /workspace - source /workspace/benchmarks/benchmark_lib.sh - - # Bridge the few metadata field names the workflow spells - # differently from what append_lm_eval_summary expects - # (PREFILL_DP_ATTN vs PREFILL_DP_ATTENTION), and fill in the - # single-engine TP/EP/DP fields from the prefill side, since - # disaggregated topologies have no single "TP" of their own. - export TP="${PREFILL_TP:-1}" - export EP_SIZE="${PREFILL_EP:-1}" - export DP_ATTENTION="${PREFILL_DP_ATTN:-false}" - export PREFILL_DP_ATTENTION="${PREFILL_DP_ATTN:-false}" - export DECODE_DP_ATTENTION="${DECODE_DP_ATTN:-false}" - - # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS})" - else - run_eval --port 30000 - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping eval artifact staging" >&2 - EVAL_FAILED=1 - else - # Files (meta_env.json, results*.json, sample*.jsonl, ...) are now in /workspace - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - # Use find for glob patterns to avoid "no match" errors - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'agent_preds.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'predictions.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'swebench_report_*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name '*.traj*' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd else - # Must run from repo root so utils/evals/${task}.yaml resolves + # Must run from repo root so utils/evals/gsm8k.yaml resolves pushd /workspace - # Source eval functions from benchmark_lib.sh source /workspace/benchmarks/benchmark_lib.sh - # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list + # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list. + # Export CONC before run_eval so meta_env.json matches validate_scores.py. if [[ -n "${EVAL_CONC:-}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) fi + export CONC="${EVAL_CONCURRENT_REQUESTS}" # Override eval context length with model's configured context_length if [[ -n "$prefill_context_length" ]]; then export EVAL_MAX_MODEL_LEN="$prefill_context_length" fi + # Metadata for append_lm_eval_summary. Prefer workflow PREFILL_EP/DECODE_EP + # and *_DP_ATTN (from job.slurm) over ENABLE_* launch booleans so DEP8/DPA + # arms record the correct topology. Set before run_eval: agentic eval-only + # recipes call append_lm_eval_summary inside run_eval(). + export TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-1}}" + export PREFILL_TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-1}}" + export PREFILL_EP="${PREFILL_EP:-1}" + [[ "${PREFILL_EP}" == "1" && "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" + export EP_SIZE="${PREFILL_EP}" + export PREFILL_NUM_WORKERS="${PREFILL_NUM_WORKERS:-${xP:-1}}" + export DECODE_TP="${DECODE_TP:-${DECODE_TP_SIZE:-1}}" + export DECODE_EP="${DECODE_EP:-1}" + [[ "${DECODE_EP}" == "1" && "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" + export DECODE_NUM_WORKERS="${DECODE_NUM_WORKERS:-${yD:-1}}" + export DP_ATTENTION="${PREFILL_DP_ATTN:-${PREFILL_ENABLE_DP:-false}}" + export PREFILL_DP_ATTENTION="${PREFILL_DP_ATTN:-${PREFILL_ENABLE_DP:-false}}" + export DECODE_DP_ATTENTION="${DECODE_DP_ATTN:-${DECODE_ENABLE_DP:-false}}" + export ISL="${BENCH_INPUT_LEN:-0}" + export OSL="${BENCH_OUTPUT_LEN:-0}" + # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, + # RESULT_FILENAME are already set via Docker -e flags from job.slurm + if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" else - # Run lm-eval against the router on port 30000 run_eval --framework lm-eval --port 30000 eval_rc=$? @@ -1177,37 +1137,18 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 EVAL_FAILED=1 else - # Set metadata env vars for append_lm_eval_summary - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, - # RESULT_FILENAME are already set via Docker -e flags from job.slurm - - append_lm_eval_summary - # Files (meta_env.json, results*.json, sample*.jsonl) are now in /workspace - - # Copy eval artifacts to run_logs for NFS extraction by runner + # Agentic eval-only: run_eval() already called append_lm_eval_summary + # with the metadata exports above. Fixed-seq-len post-bench eval needs + # an explicit append because run_eval skips it when EVAL_ONLY=false. + if [[ "${EVAL_ONLY:-false}" != "true" || "$IS_AGENTIC_RUN" != "1" ]]; then + append_lm_eval_summary + fi + EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" mkdir -p "$EVAL_COPY_DIR" for f in meta_env.json; do [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" done - # Use find for glob patterns to avoid "no match" errors find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; From 2d51b6a84ad9f3129557385136a78a68bbc94e85 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 09:51:55 +0000 Subject: [PATCH 12/30] Harden multinode agentic eval metadata and GPU pre-flight for DEP8 CI. Exclude leaky g10/g16 nodes, improve docker pre-clean and gpu_sanity drain, bridge disagg EP/DPA into lm-eval meta_env.json, and filter eval-only rows by eval-min-prefill-ep so DEP8 c128 jobs dispatch with correct metadata. Co-authored-by: Cursor --- benchmarks/benchmark_lib.sh | 151 ++++++++++++------ .../amd_utils/helpers/gpu_sanity.sh | 23 ++- benchmarks/multi_node/amd_utils/job.slurm | 13 +- .../multi_node/amd_utils/server_sglang.sh | 27 +--- benchmarks/multi_node/amd_utils/submit.sh | 17 +- perf-changelog.yaml | 2 + utils/matrix_logic/generate_sweep_configs.py | 43 +++-- .../test_generate_sweep_configs.py | 10 +- utils/matrix_logic/validation.py | 7 + utils/process_changelog.py | 26 ++- 10 files changed, 222 insertions(+), 97 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index fb329b922c..02a48eb24b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -954,44 +954,59 @@ _eval_concs_to_json() { printf '[%s]' "$joined" } -append_lm_eval_summary() { - local batch_concs="${EVAL_BATCHED_CONCS:-}" - local results_dir="${EVAL_RESULT_DIR:-}" - local out_dir="${results_dir}" - local meta_json - local metadata_conc="${CONC:-1}" - local batch_metadata="" +_env_is_true() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} - if [ -n "$batch_concs" ]; then - meta_json="./meta_env.json" - metadata_conc="${batch_concs%% *}" +_resolve_disagg_ep() { + local ep="${1:-1}" + local enable_flag="${2:-false}" + local tp_size="${3:-1}" + if [[ "$ep" == "1" ]] && _env_is_true "$enable_flag"; then + echo "$tp_size" + else + echo "$ep" + fi +} - local eval_concs_json completed_concs_json failed_concs_json - eval_concs_json=$(_eval_concs_to_json "$batch_concs") || return 1 - completed_concs_json=$( - _eval_concs_to_json "${EVAL_BATCHED_COMPLETED_CONCS:-}" - ) || return 1 - failed_concs_json=$( - _eval_concs_to_json "${EVAL_BATCHED_FAILED_CONCS:-}" - ) || return 1 - printf -v batch_metadata \ - ' "eval_concs": %s,\n "completed_eval_concs": %s,\n "failed_eval_concs": %s,\n' \ - "$eval_concs_json" \ - "$completed_concs_json" \ - "$failed_concs_json" +_normalize_bool_json() { + if _env_is_true "${1:-false}"; then + echo "true" else - if [ -z "${results_dir}" ]; then - echo "WARN: EVAL_RESULT_DIR is empty; skipping artifact collection" >&2 - return 1 - fi - if [ ! -d "${out_dir}" ]; then - echo "WARN: EVAL_RESULT_DIR='${out_dir}' does not exist; skipping artifact collection" >&2 - return 1 - fi - meta_json="${out_dir}/meta_env.json" + echo "false" fi +} + +# Export TP/EP/DP metadata for append_lm_eval_summary / meta_env.json. +# Prefer workflow PREFILL_EP/DECODE_EP and *_DP_ATTN (from job.slurm) over +# ENABLE_* launch booleans so DEP8/DPA arms record the correct topology. +bridge_disagg_eval_metadata() { + export TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-${TP:-1}}}" + export PREFILL_TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-${TP:-1}}}" + export PREFILL_EP="$(_resolve_disagg_ep "${PREFILL_EP:-1}" "${PREFILL_ENABLE_EP:-false}" "${PREFILL_TP_SIZE:-${PREFILL_TP:-1}}")" + export EP_SIZE="${PREFILL_EP}" + export PREFILL_NUM_WORKERS="${PREFILL_NUM_WORKERS:-${xP:-1}}" + export DECODE_TP="${DECODE_TP:-${DECODE_TP_SIZE:-${TP:-1}}}" + export DECODE_EP="$(_resolve_disagg_ep "${DECODE_EP:-1}" "${DECODE_ENABLE_EP:-false}" "${DECODE_TP_SIZE:-${DECODE_TP:-1}}")" + export DECODE_NUM_WORKERS="${DECODE_NUM_WORKERS:-${yD:-1}}" + + local prefill_dp="${PREFILL_DP_ATTN:-${PREFILL_DP_ATTENTION:-${PREFILL_ENABLE_DP:-false}}}" + local decode_dp="${DECODE_DP_ATTN:-${DECODE_DP_ATTENTION:-${DECODE_ENABLE_DP:-false}}}" + export DP_ATTENTION="$(_normalize_bool_json "$prefill_dp")" + export PREFILL_DP_ATTENTION="$(_normalize_bool_json "$prefill_dp")" + export DECODE_DP_ATTENTION="$(_normalize_bool_json "$decode_dp")" +} + +_write_lm_eval_meta_json() { + local meta_json="$1" + local batch_metadata="${2:-}" + local metadata_conc="${3:-${CONC:-1}}" + + bridge_disagg_eval_metadata - # Write minimal meta for collectors that expect it local model_name="${MODEL_NAME:-$MODEL}" local is_multinode_json="false" if [ "${IS_MULTINODE:-false}" = "true" ]; then @@ -1011,30 +1026,17 @@ append_lm_eval_summary() { local decode_ep="${DECODE_EP:-${EP_SIZE:-1}}" local decode_num_workers="${DECODE_NUM_WORKERS:-1}" - local dp_json="false" - if [ "${DP_ATTENTION:-false}" = "true" ]; then dp_json="true"; fi - local prefill_dp_json="$dp_json" - if [ "${PREFILL_DP_ATTENTION:-${DP_ATTENTION:-false}}" = "true" ]; then - prefill_dp_json="true" - else - prefill_dp_json="false" - fi - local decode_dp_json="$dp_json" - if [ "${DECODE_DP_ATTENTION:-${DP_ATTENTION:-false}}" = "true" ]; then - decode_dp_json="true" - else - decode_dp_json="false" - fi + local dp_json + dp_json="$(_normalize_bool_json "${DP_ATTENTION:-false}")" + local prefill_dp_json + prefill_dp_json="$(_normalize_bool_json "${PREFILL_DP_ATTENTION:-${DP_ATTENTION:-false}}")" + local decode_dp_json + decode_dp_json="$(_normalize_bool_json "${DECODE_DP_ATTENTION:-${DP_ATTENTION:-false}}")" - # Derive framework/precision from env, fallback to parsing RESULT_FILENAME - # RESULT_FILENAME format (from workflow): - # ___tp<...>_ep<...>_dpa_<...>_conc<...>_ local fw="${FRAMEWORK:-}" local prec="${PRECISION:-}" if [[ -z "$fw" || -z "$prec" ]]; then if [[ -n "${RESULT_FILENAME:-}" ]]; then - # Extract the two fields immediately before "_tp" - # Handles arbitrary underscores in exp_name by matching from the end local parsed parsed=$(echo "${RESULT_FILENAME}" | sed -n 's/.*_\([^_][^_]*\)_\([^_][^_]*\)_tp.*/\1 \2/p') local p1="${parsed%% *}" @@ -1047,6 +1049,7 @@ append_lm_eval_summary() { fi fi fi + cat > "${meta_json}" <&2 + return 1 + fi + if [ ! -d "${out_dir}" ]; then + echo "WARN: EVAL_RESULT_DIR='${out_dir}' does not exist; skipping artifact collection" >&2 + return 1 + fi + meta_json="${out_dir}/meta_env.json" + fi + + _write_lm_eval_meta_json "$meta_json" "$batch_metadata" "$metadata_conc" if [ -n "$batch_concs" ]; then echo "Prepared batched eval artifacts in: $(pwd)" diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh index 9e3df3b3be..7ac387670e 100755 --- a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh +++ b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh @@ -13,7 +13,7 @@ set -uo pipefail # Fail the node if any single GPU still has >= this many GB used. GPU_BUSY_GB="${GPU_BUSY_GB:-8}" # Poll a few times to give docker stop time to release VRAM before judging. -GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-6}" +GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-12}" if ! command -v rocm-smi >/dev/null 2>&1; then echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 @@ -21,11 +21,24 @@ if ! command -v rocm-smi >/dev/null 2>&1; then fi busy_gb=0 +busy_gpus="" for _ in $(seq "$GPU_DRAIN_TRIES"); do - used=$(rocm-smi --showmeminfo vram --json 2>/dev/null \ + if ! mapfile -t _used_bytes < <(rocm-smi --showmeminfo vram --json 2>/dev/null \ | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \ - | grep -oE '[0-9]+' | sort -n | tail -1) - busy_gb=$(( ${used:-0} / 1024 / 1024 / 1024 )) + | grep -oE '[0-9]+'); then + _used_bytes=() + fi + busy_gb=0 + busy_gpus="" + idx=0 + for used in "${_used_bytes[@]}"; do + gb=$(( used / 1024 / 1024 / 1024 )) + if (( gb >= GPU_BUSY_GB )); then + busy_gpus+=" gpu${idx}=${gb}GB" + (( busy_gb < gb )) && busy_gb=$gb + fi + idx=$((idx + 1)) + done if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then exit 0 fi @@ -33,6 +46,6 @@ for _ in $(seq "$GPU_DRAIN_TRIES"); do done echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 -echo " (threshold ${GPU_BUSY_GB}GB) -- a bare (non-containerized) process is hogging the GPU:" >&2 +echo " (threshold ${GPU_BUSY_GB}GB; busy:${busy_gpus}) -- a bare (non-containerized) process is hogging the GPU:" >&2 rocm-smi --showpids >&2 || true exit 1 diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4ca4be3785..ca055e8941 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -677,15 +677,22 @@ else fi fi # end: if ENGINE == atom-disagg -# Pre-clean (idempotent) +# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released before +# the sanity gate. stop-only left containers in Created/Exited state on some nodes. \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true +\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true +\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true +sleep 2 # GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare # (non-containerized) process hogging the GPU -- fail fast (and name it) # instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit # tears down the whole job on non-zero exit. -bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" +if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then + echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU sanity pre-flight check\" +else + bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" +fi # Start vLLM external router container on node 0 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index edb5a96158..962108183e 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1105,25 +1105,9 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { export EVAL_MAX_MODEL_LEN="$prefill_context_length" fi - # Metadata for append_lm_eval_summary. Prefer workflow PREFILL_EP/DECODE_EP - # and *_DP_ATTN (from job.slurm) over ENABLE_* launch booleans so DEP8/DPA - # arms record the correct topology. Set before run_eval: agentic eval-only - # recipes call append_lm_eval_summary inside run_eval(). - export TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-1}}" - export PREFILL_TP="${PREFILL_TP:-${PREFILL_TP_SIZE:-1}}" - export PREFILL_EP="${PREFILL_EP:-1}" - [[ "${PREFILL_EP}" == "1" && "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export EP_SIZE="${PREFILL_EP}" - export PREFILL_NUM_WORKERS="${PREFILL_NUM_WORKERS:-${xP:-1}}" - export DECODE_TP="${DECODE_TP:-${DECODE_TP_SIZE:-1}}" - export DECODE_EP="${DECODE_EP:-1}" - [[ "${DECODE_EP}" == "1" && "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${DECODE_NUM_WORKERS:-${yD:-1}}" - export DP_ATTENTION="${PREFILL_DP_ATTN:-${PREFILL_ENABLE_DP:-false}}" - export PREFILL_DP_ATTENTION="${PREFILL_DP_ATTN:-${PREFILL_ENABLE_DP:-false}}" - export DECODE_DP_ATTENTION="${DECODE_DP_ATTN:-${DECODE_ENABLE_DP:-false}}" export ISL="${BENCH_INPUT_LEN:-0}" export OSL="${BENCH_OUTPUT_LEN:-0}" + bridge_disagg_eval_metadata # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, # RESULT_FILENAME are already set via Docker -e flags from job.slurm @@ -1137,9 +1121,12 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 EVAL_FAILED=1 else - # Agentic eval-only: run_eval() already called append_lm_eval_summary - # with the metadata exports above. Fixed-seq-len post-bench eval needs - # an explicit append because run_eval skips it when EVAL_ONLY=false. + # Always rewrite meta_env.json so EP/DPA match the workflow + # topology even when run_eval() staged artifacts internally. + rewrite_lm_eval_meta_env + + # Fixed-seq-len post-bench eval still needs append to move + # results out of the temp EVAL_RESULT_DIR. if [[ "${EVAL_ONLY:-false}" != "true" || "$IS_AGENTIC_RUN" != "1" ]]; then append_lm_eval_summary fi diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 1493cc7f7f..3ce0acd2dd 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -113,9 +113,23 @@ export yD=$DECODE_WORKERS export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) export PREFILL_ENABLE_EP=${PREFILL_ENABLE_EP} export PREFILL_ENABLE_DP=${PREFILL_ENABLE_DP} +export PREFILL_TP +export PREFILL_EP=${PREFILL_EP:-1} +export PREFILL_DP_ATTN=${PREFILL_DP_ATTN:-false} +export PREFILL_NUM_WORKERS=${PREFILL_NUM_WORKERS:-$PREFILL_WORKERS} +export PREFILL_PP_SIZE=${PREFILL_PP_SIZE:-1} +export PREFILL_DCP_SIZE=${PREFILL_DCP_SIZE:-1} +export PREFILL_PCP_SIZE=${PREFILL_PCP_SIZE:-1} export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) export DECODE_ENABLE_EP=${DECODE_ENABLE_EP} export DECODE_ENABLE_DP=${DECODE_ENABLE_DP} +export DECODE_TP +export DECODE_EP=${DECODE_EP:-1} +export DECODE_DP_ATTN=${DECODE_DP_ATTN:-false} +export DECODE_NUM_WORKERS=${DECODE_NUM_WORKERS:-$DECODE_WORKERS} +export DECODE_PP_SIZE=${DECODE_PP_SIZE:-1} +export DECODE_DCP_SIZE=${DECODE_DCP_SIZE:-1} +export DECODE_PCP_SIZE=${DECODE_PCP_SIZE:-1} export DECODE_MTP_SIZE=${DECODE_MTP_SIZE} export NUM_NODES=$NUM_NODES @@ -172,7 +186,8 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" +# g10/g16: recurring bare-GPU VRAM leaks fail gpu_sanity pre-flight (see PR #2308). +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15,mia1-p01-g16}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e178816066..93814b0c82 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5170,3 +5170,5 @@ - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 + evals-only: true + eval-min-prefill-ep: 8 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 0a1a0e88a8..008f2827dd 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -191,6 +191,35 @@ def agentic_kv_offload_suffix( return f"kv{kv_offloading}-{kv_offload_backend['name']}" +def multinode_agentic_exp_name( + model_code: str, + prefill: dict, + decode: dict, + conc_batch: list[int], + offload_suffix: str, +) -> str: + """Build a multinode agentic exp-name that encodes topology, not just TP.""" + + def _worker_tag(worker: dict, role_prefix: str) -> str: + ep = worker.get(Fields.EP.value, 1) + dpa = worker.get(Fields.DP_ATTN.value, False) + tag = ( + f"{role_prefix}{worker[Fields.NUM_WORKER.value]}" + f"x{worker[Fields.TP.value]}" + ) + if ep != 1: + tag += f"ep{ep}" + if dpa: + tag += "dpa" + return tag + + return ( + f"{model_code}_{_worker_tag(prefill, 'p')}_{_worker_tag(decode, 'd')}" + f"_conc{'x'.join(str(c) for c in conc_batch)}" + f"{offload_suffix}" + ) + + def component_metadata(benchmark: dict, config: dict) -> dict: """Resolve optional component metadata from its validated scope.""" metadata = {} @@ -827,11 +856,8 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.KV_OFFLOADING.value: kv_offloading, Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, - Fields.EXP_NAME.value: ( - f"{model_code}_p{prefill[Fields.NUM_WORKER.value]}x{prefill[Fields.TP.value]}" - f"_d{decode[Fields.NUM_WORKER.value]}x{decode[Fields.TP.value]}" - f"_conc{'x'.join(str(c) for c in conc_batch)}" - f"{offload_suffix}" + Fields.EXP_NAME.value: multinode_agentic_exp_name( + model_code, prefill, decode, conc_batch, offload_suffix ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", @@ -1125,11 +1151,8 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.KV_OFFLOADING.value: kv_offloading, Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, - Fields.EXP_NAME.value: ( - f"{model_code}_p{prefill[Fields.NUM_WORKER.value]}x{prefill[Fields.TP.value]}" - f"_d{decode[Fields.NUM_WORKER.value]}x{decode[Fields.TP.value]}" - f"_conc{'x'.join(str(c) for c in conc_batch)}" - f"{offload_suffix}" + Fields.EXP_NAME.value: multinode_agentic_exp_name( + model_code, prefill, decode, conc_batch, offload_suffix ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index f5279ce2d5..da91222266 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2301,11 +2301,11 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): assert len(result) == 5 assert [entry["conc"] for entry in result] == [[16], [32], [64], [128], [256]] assert [entry["exp-name"] for entry in result] == [ - "dsv4_p2x4_d1x4_conc16", - "dsv4_p2x4_d1x4_conc32", - "dsv4_p2x4_d1x4_conc64", - "dsv4_p2x4_d1x4_conc128", - "dsv4_p2x4_d1x4_conc256", + "dsv4_p2x4ep4_d1x4_conc16", + "dsv4_p2x4ep4_d1x4_conc32", + "dsv4_p2x4ep4_d1x4_conc64", + "dsv4_p2x4ep4_d1x4_conc128", + "dsv4_p2x4ep4_d1x4_conc256", ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 1b0efc8a19..79a5a9a1ff 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -875,6 +875,13 @@ class ChangelogEntry(BaseModel): pr_link: str = Field(alias="pr-link") evals_only: bool = Field(alias="evals-only", default=False) all_evals: bool = Field(alias="all-evals", default=False) + eval_min_prefill_ep: Optional[int] = Field( + alias="eval-min-prefill-ep", default=None, ge=1, + description=( + "When set, multinode eval rows whose prefill.ep is below this " + "threshold are dropped after eval selection." + ), + ) scenario_type: Optional[List[Literal["fixed-seq-len", "agentic-coding"]]] = Field( alias="scenario-type", default=None, min_length=1, description="Restrict to specific scenario types (e.g., ['fixed-seq-len', 'agentic-coding'])" diff --git a/utils/process_changelog.py b/utils/process_changelog.py index b885b9b5f7..a1bbe7f78c 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -84,6 +84,26 @@ def trim_conc(entries: list[dict]) -> list[dict]: return [e for i, e in enumerate(out) if i not in drop] +def filter_eval_rows_by_prefill_ep( + eval_rows: list[dict], min_prefill_ep: int | None +) -> list[dict]: + """Drop multinode eval rows below a prefill EP threshold.""" + if min_prefill_ep is None: + return eval_rows + kept: list[dict] = [] + for row in eval_rows: + prefill = row.get("prefill") + if isinstance(prefill, dict): + ep = prefill.get("ep", 1) + try: + if int(ep) < min_prefill_ep: + continue + except (TypeError, ValueError): + continue + kept.append(row) + return kept + + def get_config_keys_from_master( config_keys: list[str], master_config: dict ) -> list[str]: @@ -255,7 +275,11 @@ def main(): except subprocess.CalledProcessError as e: print(e.stderr) raise - all_eval_results.extend(json.loads(eval_result.stdout)) + entry_eval_results = json.loads(eval_result.stdout) + entry_eval_results = filter_eval_rows_by_prefill_ep( + entry_eval_results, entry.eval_min_prefill_ep + ) + all_eval_results.extend(entry_eval_results) if args.trim_conc: all_benchmark_results = trim_conc(all_benchmark_results) From 6253ce94ddccad906272d4d197ed3ad56420b9f7 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 14:04:04 +0000 Subject: [PATCH 13/30] Add EP1 GSM8K eval changelog entry alongside DEP8 c128 eval. Append a second evals-only changelog row filtered with eval-max-prefill-ep: 1 so TP8/EP1 low-conc (eval-conc 32) runs separately from the existing DEP8/DPA c128 eval, matching one-eval-per-topology policy without mutating prior entries. Co-authored-by: Cursor --- perf-changelog.yaml | 8 ++++++++ utils/matrix_logic/validation.py | 7 +++++++ utils/process_changelog.py | 19 +++++++++++++------ utils/test_process_changelog.py | 10 ++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93814b0c82..b4ae36fee9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5172,3 +5172,11 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 evals-only: true eval-min-prefill-ep: 8 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "GSM8K lm-eval for TP8/EP1 low-conc hicache arm at eval-conc 32; complements the DEP8/DPA c128 eval above (one eval job per distinct parallelism topology)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 + evals-only: true + eval-max-prefill-ep: 1 diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 79a5a9a1ff..15d4e0fe94 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -882,6 +882,13 @@ class ChangelogEntry(BaseModel): "threshold are dropped after eval selection." ), ) + eval_max_prefill_ep: Optional[int] = Field( + alias="eval-max-prefill-ep", default=None, ge=1, + description=( + "When set, multinode eval rows whose prefill.ep is above this " + "threshold are dropped after eval selection." + ), + ) scenario_type: Optional[List[Literal["fixed-seq-len", "agentic-coding"]]] = Field( alias="scenario-type", default=None, min_length=1, description="Restrict to specific scenario types (e.g., ['fixed-seq-len', 'agentic-coding'])" diff --git a/utils/process_changelog.py b/utils/process_changelog.py index a1bbe7f78c..c5e128c30f 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -85,10 +85,12 @@ def trim_conc(entries: list[dict]) -> list[dict]: def filter_eval_rows_by_prefill_ep( - eval_rows: list[dict], min_prefill_ep: int | None + eval_rows: list[dict], + min_prefill_ep: int | None = None, + max_prefill_ep: int | None = None, ) -> list[dict]: - """Drop multinode eval rows below a prefill EP threshold.""" - if min_prefill_ep is None: + """Drop multinode eval rows outside a prefill EP range.""" + if min_prefill_ep is None and max_prefill_ep is None: return eval_rows kept: list[dict] = [] for row in eval_rows: @@ -96,10 +98,13 @@ def filter_eval_rows_by_prefill_ep( if isinstance(prefill, dict): ep = prefill.get("ep", 1) try: - if int(ep) < min_prefill_ep: - continue + ep_val = int(ep) except (TypeError, ValueError): continue + if min_prefill_ep is not None and ep_val < min_prefill_ep: + continue + if max_prefill_ep is not None and ep_val > max_prefill_ep: + continue kept.append(row) return kept @@ -277,7 +282,9 @@ def main(): raise entry_eval_results = json.loads(eval_result.stdout) entry_eval_results = filter_eval_rows_by_prefill_ep( - entry_eval_results, entry.eval_min_prefill_ep + entry_eval_results, + entry.eval_min_prefill_ep, + entry.eval_max_prefill_ep, ) all_eval_results.extend(entry_eval_results) diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 7f6316b615..be879e7309 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -702,3 +702,13 @@ def fake_run(command, **kwargs): assert [r["exp-name"] for r in output["multinode_agentic_evals"]] == ["multinode_agentic_eval"] assert output["evals"] == [] assert output["agentic_evals"] == [] + + +def test_filter_eval_rows_by_prefill_ep_range(): + rows = [ + {"prefill": {"ep": 1}, "eval-conc": 32}, + {"prefill": {"ep": 8}, "eval-conc": 128}, + ] + assert process_changelog.filter_eval_rows_by_prefill_ep(rows, min_prefill_ep=8) == [rows[1]] + assert process_changelog.filter_eval_rows_by_prefill_ep(rows, max_prefill_ep=1) == [rows[0]] + assert process_changelog.filter_eval_rows_by_prefill_ep(rows) == rows From 2393662ea344bc7cc66310b29f2a6ec1422aa3b1 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 14:09:13 +0000 Subject: [PATCH 14/30] Revert "Add EP1 GSM8K eval changelog entry alongside DEP8 c128 eval." This reverts commit b9d85e5960d25a77b97b223b9088df88ddb6689b. --- perf-changelog.yaml | 9 --------- utils/matrix_logic/validation.py | 7 ------- utils/process_changelog.py | 19 ++++++------------- utils/test_process_changelog.py | 10 ---------- 4 files changed, 6 insertions(+), 39 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b4ae36fee9..d4e61dcf95 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5170,13 +5170,4 @@ - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 - evals-only: true - eval-min-prefill-ep: 8 -- config-keys: - - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache - description: - - "GSM8K lm-eval for TP8/EP1 low-conc hicache arm at eval-conc 32; complements the DEP8/DPA c128 eval above (one eval job per distinct parallelism topology)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 - evals-only: true - eval-max-prefill-ep: 1 diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 15d4e0fe94..79a5a9a1ff 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -882,13 +882,6 @@ class ChangelogEntry(BaseModel): "threshold are dropped after eval selection." ), ) - eval_max_prefill_ep: Optional[int] = Field( - alias="eval-max-prefill-ep", default=None, ge=1, - description=( - "When set, multinode eval rows whose prefill.ep is above this " - "threshold are dropped after eval selection." - ), - ) scenario_type: Optional[List[Literal["fixed-seq-len", "agentic-coding"]]] = Field( alias="scenario-type", default=None, min_length=1, description="Restrict to specific scenario types (e.g., ['fixed-seq-len', 'agentic-coding'])" diff --git a/utils/process_changelog.py b/utils/process_changelog.py index c5e128c30f..a1bbe7f78c 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -85,12 +85,10 @@ def trim_conc(entries: list[dict]) -> list[dict]: def filter_eval_rows_by_prefill_ep( - eval_rows: list[dict], - min_prefill_ep: int | None = None, - max_prefill_ep: int | None = None, + eval_rows: list[dict], min_prefill_ep: int | None ) -> list[dict]: - """Drop multinode eval rows outside a prefill EP range.""" - if min_prefill_ep is None and max_prefill_ep is None: + """Drop multinode eval rows below a prefill EP threshold.""" + if min_prefill_ep is None: return eval_rows kept: list[dict] = [] for row in eval_rows: @@ -98,13 +96,10 @@ def filter_eval_rows_by_prefill_ep( if isinstance(prefill, dict): ep = prefill.get("ep", 1) try: - ep_val = int(ep) + if int(ep) < min_prefill_ep: + continue except (TypeError, ValueError): continue - if min_prefill_ep is not None and ep_val < min_prefill_ep: - continue - if max_prefill_ep is not None and ep_val > max_prefill_ep: - continue kept.append(row) return kept @@ -282,9 +277,7 @@ def main(): raise entry_eval_results = json.loads(eval_result.stdout) entry_eval_results = filter_eval_rows_by_prefill_ep( - entry_eval_results, - entry.eval_min_prefill_ep, - entry.eval_max_prefill_ep, + entry_eval_results, entry.eval_min_prefill_ep ) all_eval_results.extend(entry_eval_results) diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index be879e7309..7f6316b615 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -702,13 +702,3 @@ def fake_run(command, **kwargs): assert [r["exp-name"] for r in output["multinode_agentic_evals"]] == ["multinode_agentic_eval"] assert output["evals"] == [] assert output["agentic_evals"] == [] - - -def test_filter_eval_rows_by_prefill_ep_range(): - rows = [ - {"prefill": {"ep": 1}, "eval-conc": 32}, - {"prefill": {"ep": 8}, "eval-conc": 128}, - ] - assert process_changelog.filter_eval_rows_by_prefill_ep(rows, min_prefill_ep=8) == [rows[1]] - assert process_changelog.filter_eval_rows_by_prefill_ep(rows, max_prefill_ep=1) == [rows[0]] - assert process_changelog.filter_eval_rows_by_prefill_ep(rows) == rows From b131cb9cddf5ff93899f30a92442dca076622d28 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 14:29:37 +0000 Subject: [PATCH 15/30] include node mia1-p01-g10 and mia1-p01-g16 Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/submit.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 3ce0acd2dd..487358b436 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,8 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -# g10/g16: recurring bare-GPU VRAM leaks fail gpu_sanity pre-flight (see PR #2308). -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15,mia1-p01-g16}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi From 3e7bb76f9dda5a66751bd95a53bee35948450865 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 14:38:34 +0000 Subject: [PATCH 16/30] exclude mia1-p01-g10 Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/submit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 487358b436..4222fb0d1b 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,7 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi From 4aa0713f398f693eef89fc29b706419b071a192d Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 28 Jul 2026 15:26:55 +0000 Subject: [PATCH 17/30] add con=1 in config Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e2b8ac2b53..5b670253fe 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2262,7 +2262,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: "none" - conc-list: [ 2,4,8,16,32 ] + conc-list: [ 1,2,4,8,16,32 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 2b395cb5c9c4347f268280d5d3c460cd7d907768 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 29 Jul 2026 03:15:23 +0000 Subject: [PATCH 18/30] remove con=64 Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 5b670253fe..c4a1ed608c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2288,7 +2288,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - spec-decoding: "none" - conc-list: [ 64, 96, 128 ] + conc-list: [ 96, 128 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 3b30737878577cc0a1e73fe9d78fd14341bd564a Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 29 Jul 2026 03:16:54 +0000 Subject: [PATCH 19/30] remove SGLANG_AITER_MLA_PERSIST Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/job.slurm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index ca055e8941..3c8a6f16a8 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -458,8 +458,7 @@ DOCKER_ENV_COMMON=( -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN:-false} -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} - -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} - -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-} -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} -e DURATION=\${DURATION:-1800} -e IS_AGENTIC=\${IS_AGENTIC:-0} From 2ad6af3d899604f26978a7d9e656eb44b617665d Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 21 Jul 2026 06:44:41 +0000 Subject: [PATCH 20/30] fix(dsv4): use native EAGLE MTP flags without draft-model-path for AgentX DeepSeek-V4's MTP head is a native NextN/EAGLE draft (shipped with the model), so SGLang defaults the draft path to the main checkpoint and does not need --speculative-draft-model-path. Adds mtp_flags matching the upstream SGLang DeepSeek-V4 AMD recipe (EAGLE, topk=1); num-steps and num-draft-tokens are already derived from DECODE_MTP_SIZE in server_sglang.sh. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index f9e06ac120..2231e6fe60 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -367,6 +367,7 @@ DeepSeek-V4-Pro-AgentX: base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" dp_flags: "--enable-dp-attention --enable-prefill-delayer" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" + mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" prefill: mem_fraction_static: 0.72 disable_radix_cache: false From 3e2992bde8b14a97a58437951ce8ece470466881 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 10:25:55 +0000 Subject: [PATCH 21/30] add dsv4 disagg mtp config. Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 67 +++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c4a1ed608c..a27200bd04 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2272,12 +2272,6 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: dp-attn: false additional-settings: - "PREFILL_NODES=1" - # Node-0 sibling client: run the aiperf trace-replay in its own sibling - # container on node 0 (via the host docker socket) instead of inside the - # server container. Reuse the (publicly pullable) server image so - # upstream CI can fetch it; aiperf is built on the fly from - # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) - # selects the sibling-container mode. - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" decode: num-worker: 1 @@ -2298,12 +2292,6 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: dp-attn: true additional-settings: - "PREFILL_NODES=1" - # Node-0 sibling client: run the aiperf trace-replay in its own sibling - # container on node 0 (via the host docker socket) instead of inside the - # server container. Reuse the (publicly pullable) server image so - # upstream CI can fetch it; aiperf is built on the fly from - # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) - # selects the sibling-container mode. - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" decode: num-worker: 1 @@ -2314,6 +2302,61 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "mtp" + conc-list: [ 2,4,8,16,32 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 64, 96, 128 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + # DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. # EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no # gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the From 51909c4674f23b2da8fcc6208becb0ddb4dce85c Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 23 Jul 2026 10:34:20 +0000 Subject: [PATCH 22/30] Add perf-changelog.yaml entry for dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Required per AGENTS.md's "Adding a benchmark configuration" rule so the new EAGLE/MTP arm actually triggers a benchmark sweep on merge. 中文：为 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp 添加 perf-changelog.yaml 条目。根据 AGENTS.md 的"添加基准测试配置"规则，需要此条目 才能在合并时为新的 EAGLE/MTP 分支触发基准测试扫描。 Co-authored-by: Cursor --- perf-changelog.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d4e61dcf95..bfb8638696 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5171,3 +5171,10 @@ - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + description: + - "Add EAGLE/MTP speculative-decoding arm for the DSv4 MI355X agentic disagg HiCache benchmark, mirroring the existing dsv4-fp4-mi355x-sglang-disagg-agentic-hicache TP8/EP1 and TP8/EP8/DPA topologies with spec-decoding=mtp and DECODE_MTP_SIZE=3." + - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed." + - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309 From 47f715021fc296f78909d7a52235aad62fa4f786 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 24 Jul 2026 08:48:13 +0000 Subject: [PATCH 23/30] extend gpu sanity check wait timeout Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh index 7ac387670e..2a67483afa 100755 --- a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh +++ b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh @@ -42,7 +42,7 @@ for _ in $(seq "$GPU_DRAIN_TRIES"); do if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then exit 0 fi - sleep 5 + sleep 30 done echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 From e2c1e38148073ae40db48778af22cb372836267b Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Sun, 26 Jul 2026 14:16:08 +0000 Subject: [PATCH 24/30] fix(dsv4): simulate MTP acceptance at the golden AL for agentic runs Addresses PR #2309 review feedback (functionstackx): https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624 Agentic trace replay doesn't reproduce real token-by-token traffic, so measured MTP/EAGLE acceptance there isn't representative of production. Per the AgentX fairness guidelines, wire SGLANG_SIMULATE_ACC_LEN (pinned to the committed golden AL in golden_al_distribution/dsv4_mtp.yaml, thinking_on, num_speculative_tokens=3 -> 2.49), SGLANG_SIMULATE_ACC_METHOD and SGLANG_SIMULATE_ACC_TOKEN_MODE into the decode launch command for agentic MTP runs, matching the existing GB300 dynamo-sglang DSv4 MTP recipes. Falls back to real acceptance with a warning if a future draft length has no committed golden AL wired in yet. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 962108183e..c706eb6cfe 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1273,7 +1273,29 @@ else DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" fi set +x - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ + + # Agentic trace replay doesn't reproduce real token-by-token traffic, so + # measured MTP/EAGLE acceptance there isn't representative (PR #2309 + # review: https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624). + # Per the AgentX fairness guidelines (golden_al_distribution/README.md), + # agentic spec-decode runs must instead simulate acceptance at the + # model's committed golden AL for this draft length instead of measuring + # real (and non-representative) acceptance. Golden curve source: + # golden_al_distribution/dsv4_mtp.yaml (thinking_on). + DECODE_SIM_ACC_ENV="" + if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC:-0}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then + DSV4_GOLDEN_AL="" + case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in + *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; + esac + if [[ -n "$DSV4_GOLDEN_AL" ]]; then + DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" + else + echo "WARNING: agentic MTP run (model=${MODEL_NAME}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 + fi + fi + + DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${DECODE_SIM_ACC_ENV} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path ${MODEL_DIR}/${MODEL_NAME} \ --disaggregation-mode decode \ --disaggregation-ib-device ${IBDEVICES} \ From 3f47b7d8769dfa268a93ecb53ad53a7aa7c08ad3 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 27 Jul 2026 00:48:39 +0000 Subject: [PATCH 25/30] fix(dsv4): wire golden AL for MTP draft lengths 1 and 2 Extend the simulated-acceptance lookup to cover DECODE_MTP_SIZE=1 and =2 using the committed thinking_on curve in golden_al_distribution/dsv4_mtp.yaml (1 -> 1.79, 2 -> 2.27), matching the existing size=3 -> 2.49 entry. Sizes without a wired golden AL still fall back to real acceptance with a warning. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/server_sglang.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index c706eb6cfe..b12492412d 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1286,6 +1286,8 @@ else if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC:-0}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then DSV4_GOLDEN_AL="" case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in + *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; + *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; esac if [[ -n "$DSV4_GOLDEN_AL" ]]; then From 30f75955aa7fb0e9350ace6168370709b835e78b Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 29 Jul 2026 14:43:30 +0000 Subject: [PATCH 26/30] fix(dsv4): disable synthetic MTP during eval runs Use golden-AL simulation only for agentic throughput benchmarks; turn it off when RUN_EVAL or EVAL_ONLY is set so GSM8K scores reflect real MTP acceptance. Align hicache-mtp conc lists with DPA (add c1, drop c64). Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 29 +++++++++++-------- configs/amd-master.yaml | 4 +-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index b12492412d..1ea54b3745 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1278,22 +1278,27 @@ else # measured MTP/EAGLE acceptance there isn't representative (PR #2309 # review: https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624). # Per the AgentX fairness guidelines (golden_al_distribution/README.md), - # agentic spec-decode runs must instead simulate acceptance at the - # model's committed golden AL for this draft length instead of measuring - # real (and non-representative) acceptance. Golden curve source: + # agentic throughput benchmarks simulate acceptance at the model's + # committed golden AL instead of measuring real (non-representative) + # acceptance. Eval runs (RUN_EVAL / EVAL_ONLY) need real acceptance so + # GSM8K scores reflect actual MTP behavior. Golden curve source: # golden_al_distribution/dsv4_mtp.yaml (thinking_on). DECODE_SIM_ACC_ENV="" if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC:-0}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then - DSV4_GOLDEN_AL="" - case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in - *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; - *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; - *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; - esac - if [[ -n "$DSV4_GOLDEN_AL" ]]; then - DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" + if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then + echo "[INFO] Eval mode: synthetic MTP disabled (using real acceptance)" else - echo "WARNING: agentic MTP run (model=${MODEL_NAME}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 + DSV4_GOLDEN_AL="" + case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in + *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; + *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; + *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; + esac + if [[ -n "$DSV4_GOLDEN_AL" ]]; then + DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" + else + echo "WARNING: agentic MTP run (model=${MODEL_NAME}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 + fi fi fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a27200bd04..d345e4a789 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2317,7 +2317,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - dram-utilization: 0.80 search-space: - spec-decoding: "mtp" - conc-list: [ 2,4,8,16,32 ] + conc-list: [ 1,2,4,8,16,32 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -2337,7 +2337,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=3" - spec-decoding: "mtp" - conc-list: [ 64, 96, 128 ] + conc-list: [ 96, 128 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 4ec17746888c84bcaa9cae95897ac04004a78f5e Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 29 Jul 2026 15:13:37 +0000 Subject: [PATCH 27/30] remove dsv4-fp4-mi355x-sglang-disagg-agentic-hicache from perf-changelog Signed-off-by: Theresa Shan --- perf-changelog.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2926cbf662..a9774ffc67 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5168,17 +5168,6 @@ - "Measured behaviour: below conc 8 the GPU-resident and DRAM arms are within run-to-run noise (1-5%). At conc 16 and 24 the GPU-resident arm thrashes -- prefix cache hit rate 2.7%, TTFT p50 86s and 191s, 49.6 and 54.9 output tok/s -- because the working set exceeds GPU KV and prefixes are recomputed. The DRAM arm holds TTFT p50 0.85s and 6.2s for 245.0 and 260.6 output tok/s (4-5x), with the CPU tier serving a 62% external prefix cache hit rate at conc 24. The high-conc GPU-resident points are retained deliberately as the honest baseline that makes the offload gain legible." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2371 -- config-keys: - - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache - description: - - "Bump image lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 -> v0.5.15.post1-rocm720-mi35x-20260719." - - "HiCache sizing: recipe now sets HICACHE_RATIO=4 and FORCE_HICACHE_RATIO=1 to explicitly opt into ratio-based sizing (DeepSeek-V4's hybrid HiCache pool rejects --hicache-size); job.slurm now forwards FORCE_HICACHE_RATIO into the container HiCache env file, where it was previously silently dropped." - - "Decode cuda-graph crash workaround: disable prefill cuda-graph for DeepSeek-V4-Pro-AgentX, drop --enable-two-batch-overlap from dp_flags, lower prefill mem_fraction_static 0.8->0.72 and no-dp max_running_requests 128->64; add TORCH_NCCL_BLOCKING_WAIT/NCCL_BLOCKING_WAIT=1 to avoid a ROCm 7.2 HIP-runtime watchdog race during decode cuda-graph capture." - - "Fix router policy env-var bug: server_sglang.sh read ROUTER_PREFILL_POLICY but recipes export PREFILL_ROUTER_POLICY, so the recipe override was silently ignored; also switch the agentic default prefill router policy cache_aware -> consistent_hashing and add --dp-aware (superseding the separate --decode-policy override)." - - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." - - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308 - - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp description: From 7d23a572e1c348dd63fef036f4c4f23cc05847f6 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 29 Jul 2026 15:27:38 +0000 Subject: [PATCH 28/30] fix(amd): exclude mia1-p01-g16 from default SLURM node list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node g16 repeatedly fails GPU sanity pre-flight (VRAM held by stale processes after container stop); exclude it by default like g09–g15. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/submit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 4222fb0d1b..08e7607069 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,7 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15,mia1-p01-g16}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi From 62555418da20ce0d2ded0e25801008cbacecd767 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 30 Jul 2026 06:59:35 +0000 Subject: [PATCH 29/30] fix(amd): extend GPU drain poll and trim default node excludes Double gpu_sanity VRAM drain retries so docker teardown has more time to release memory before pre-flight fails. Drop g14-g16 from the default SLURM exclude list now that those nodes are schedulable again. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh | 2 +- benchmarks/multi_node/amd_utils/submit.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh index 2a67483afa..9db3331deb 100755 --- a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh +++ b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh @@ -13,7 +13,7 @@ set -uo pipefail # Fail the node if any single GPU still has >= this many GB used. GPU_BUSY_GB="${GPU_BUSY_GB:-8}" # Poll a few times to give docker stop time to release VRAM before judging. -GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-12}" +GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-24}" if ! command -v rocm-smi >/dev/null 2>&1; then echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 08e7607069..6a1598d81d 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -186,7 +186,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15,mia1-p01-g16}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi From 8787b7a7af71ba97727915caf5f8d07fab937481 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 30 Jul 2026 08:29:29 +0000 Subject: [PATCH 30/30] fix(agentic): use explicit Python 3.11 for isolated aiperf venv aiperf>=0.12.0 requires Python >=3.11,<3.14 (dropped 3.10 support in SemiAnalysisAI/aiperf#1107). The sglang-rocm/vllm-rocm container images ship Python 3.10.12 as the default python3, which caused `uv pip install` to fail silently during venv creation, leaving the aiperf/hf CLIs uninstalled and causing agentic trace replay to fail with exit 127. Changed `uv venv --python "$(command -v python3)"` to `uv venv --python "${AIPERF_PYTHON_VERSION:-3.11}"` to allow uv's auto-download of a standalone Python 3.11 interpreter when the system lacks one. This makes agentic benchmarks work on any container image that has uv available, regardless of bundled Python version. Co-authored-by: Cursor --- benchmarks/benchmark_lib.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index d34993f590..51426faf3f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1675,8 +1675,19 @@ install_agentic_deps() { rm -rf "$AIPERF_VENV" mkdir -p "$AIPERF_UV_CACHE_DIR" + # Request an explicit interpreter version rather than binding to whatever + # `python3` resolves to in the server container. aiperf's pyproject.toml + # dropped Python 3.10 support (SemiAnalysisAI/aiperf#1107); the sglang-rocm + # /vllm-rocm images still ship 3.10.12 as their default python3, so + # `--python "$(command -v python3)"` pinned the venv to an interpreter that + # can no longer satisfy `requires-python = ">=3.11,<3.14"`, leaving the venv + # without aiperf/hf installed (silent until the aiperf/hf calls below hit + # "No such file or directory"). uv auto-downloads a standalone build of the + # requested version when the system doesn't have one (same network path + # already used to fetch uv itself above), so this doesn't depend on the + # container image bundling a new-enough Python. UV_CACHE_DIR="$AIPERF_UV_CACHE_DIR" \ - "$AIPERF_UV_BIN" venv --python "$(command -v python3)" "$AIPERF_VENV" + "$AIPERF_UV_BIN" venv --python "${AIPERF_PYTHON_VERSION:-3.11}" "$AIPERF_VENV" UV_CACHE_DIR="$AIPERF_UV_CACHE_DIR" \ "$AIPERF_UV_BIN" pip install --python "$AIPERF_PYTHON" \ -r "$AGENTIC_DIR/requirements.txt" \