From e868f8cca3363c2f8468ac7a1932a24d9ee5ff7b Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 17 Jul 2026 10:25:49 +0900 Subject: [PATCH 01/35] [AMD][MI355X] Add DSv4 FP4 agentic MTP recipe with DPA tuning - New config dsv4-fp4-mi355x-sglang-agentic-mtp (separate from hicache) - Image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 - EAGLE spec decoding: 3 steps, topk 1, 4 draft tokens - DPA: prefill delayer, GPU_MAX_HW_QUEUES=5, chunked prefill 8192*TP - MAX_RUNNING_REQUESTS=2*CONC, CUDA_GRAPH_MAX_BS=CONC (cap 128) - MEM_FRACTION_STATIC reduced by 0.10 for spec decoding headroom - Sweep TP8 conc [4,8] non-DPA + conc [16,32,48,52] DPA Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 205 ++++++++++++++++++ configs/amd-master.yaml | 31 +++ perf-changelog.yaml | 11 + 3 files changed, 247 insertions(+) create mode 100755 benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh new file mode 100755 index 0000000000..612f2c0f5c --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using SGLang. +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires one of these. +# KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# ROCR/HIP visibility under slurm cgroups. +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +if agentic_kv_offload_enabled; then + # HiCache config — https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4 + case "$KV_OFFLOAD_BACKEND" in + hicache) + HICACHE_RATIO=4 + HICACHE_WRITE_POLICY="write_through" + HICACHE_IO_BACKEND="direct" + HICACHE_MEM_LAYOUT="page_first_direct" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) + echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: hicache)" >&2 + exit 1 + ;; + esac +fi +# ---- Client config ---------------------------------------------------------- +export AIPERF_HTTP_TCP_USER_TIMEOUT=1000000 + +# ---- LLM server config ---------------------------------------------------------- +USE_SGLANG_ROUTER=false +SGLANG_BACKEND_PORT="$PORT" +ROUTER_LOG="$RESULT_DIR/router.log" +MEM_FRACTION_STATIC=0.90 +CHUNKED_PREFILL_SIZE=8192 +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "$DP_ATTENTION" = "true" ]; then + USE_SGLANG_ROUTER=true + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + SGLANG_BACKEND_PORT=$((PORT + 1)) + SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) + SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) + + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + + CHUNKED_PREFILL_SIZE=$((8192 * TP)) + PARALLEL_ARGS+=( + --dp "$TP" + --enable-dp-attention + --enable-prefill-delayer + ) +fi + +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--ep-size "$EP_SIZE") +fi + +# SGLang treats max-running-requests as a global DPA limit and partitions it +# internally. CUDA graph capture is per scheduler, so only its batch size is +# divided across DP ranks. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS=$CONC +[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 + +# Simulated acceptance-length (AL) settings. +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high +export SGLANG_SIMULATE_ACC_LEN=2.49 +export SGLANG_SIMULATE_ACC_METHOD=match-expected +export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token + +export SGLANG_USE_ROCM700A=0 +export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton +export AITER_BF16_FP8_MOE_BOUND=0 + +export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + +METRICS_ARGS=(--enable-metrics) +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +if [ ${#SPEC_ARGS[@]} -gt 0 ]; then + MEM_FRACTION_STATIC=$(awk "BEGIN {printf \"%.2f\", $MEM_FRACTION_STATIC - 0.10}") +fi + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$SGLANG_BACKEND_PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend compressed + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --swa-full-tokens-ratio 0.10 + --page-size 256 + --kv-cache-dtype fp8_e4m3 + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --disable-shared-experts-fusion + --tool-call-parser deepseekv4 + --reasoning-parser deepseek-v4 + --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" + --watchdog-timeout 1800 + "${METRICS_ARGS[@]}" + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" + +{ + echo "=== SGLANG_* env vars at launch ===" + env | grep -E '^SGLANG_' | sort + echo "===================================" +} | tee "$SERVER_LOG" + +echo "Starting SGLang server for MI355X..." +"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_SGLANG_ROUTER" = "true" ]; then + echo "Starting SGLang router on port $PORT for $TP DP ranks..." + "${SGLANG_ROUTER_CMD[@]}" \ + --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ + --policy consistent_hashing \ + --request-id-headers x-correlation-id \ + --dp-aware \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ + --connect-timeout-secs 900 \ + --request-timeout-secs 14400 \ + --disable-health-check \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" +REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ac4e276906..f70ca1a74e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1551,6 +1551,37 @@ dsv4-fp4-mi355x-sglang-agentic-hicache: - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64] } - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } +dsv4-fp4-mi355x-sglang-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8] } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64] } + - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } + +dsv4-fp4-mi355x-sglang-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 52], spec-decoding: mtp } + # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 892e3ee2b6..ce5bf3608e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4974,3 +4974,14 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + description: + - "Add DSv4 FP4 MI355X SGLang agentic MTP recipe (separate from hicache config)" + - "Image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714" + - "EAGLE speculative decoding: 3 steps, topk 1, 4 draft tokens" + - "DPA: --enable-prefill-delayer, GPU_MAX_HW_QUEUES=5, chunked prefill 8192*TP" + - "MAX_RUNNING_REQUESTS = 2*CONC, CUDA_GRAPH_MAX_BS = CONC (cap 128)" + - "MEM_FRACTION_STATIC reduced by 0.10 when spec decoding active" + - "Sweep TP8 conc [4,8] non-DPA + conc [16,32,48,52] DPA" From 281087a78a1d238582620835e6b478ca76d84a0b Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 17 Jul 2026 10:29:03 +0900 Subject: [PATCH 02/35] fix(changelog): add pr-link for dsv4-fp4-mi355x-sglang-agentic-mtp entry Co-Authored-By: Claude Opus 4.6 --- perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ce5bf3608e..784be2d9c1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4985,3 +4985,4 @@ - "MAX_RUNNING_REQUESTS = 2*CONC, CUDA_GRAPH_MAX_BS = CONC (cap 128)" - "MEM_FRACTION_STATIC reduced by 0.10 when spec decoding active" - "Sweep TP8 conc [4,8] non-DPA + conc [16,32,48,52] DPA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2254 From 04529c77f901044dc1f9bbe23426362f45b005a5 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 17 Jul 2026 10:34:58 +0900 Subject: [PATCH 03/35] fix(changelog): trim verbose description for agentic-mtp entry Co-Authored-By: Claude Opus 4.6 --- perf-changelog.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 784be2d9c1..60f0513ce6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4979,10 +4979,4 @@ - dsv4-fp4-mi355x-sglang-agentic-mtp description: - "Add DSv4 FP4 MI355X SGLang agentic MTP recipe (separate from hicache config)" - - "Image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714" - - "EAGLE speculative decoding: 3 steps, topk 1, 4 draft tokens" - - "DPA: --enable-prefill-delayer, GPU_MAX_HW_QUEUES=5, chunked prefill 8192*TP" - - "MAX_RUNNING_REQUESTS = 2*CONC, CUDA_GRAPH_MAX_BS = CONC (cap 128)" - - "MEM_FRACTION_STATIC reduced by 0.10 when spec decoding active" - - "Sweep TP8 conc [4,8] non-DPA + conc [16,32,48,52] DPA" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2254 From 83530a75e320c8d60126e8f141063f562c2e7f76 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 17 Jul 2026 10:37:27 +0900 Subject: [PATCH 04/35] fix(config): remove duplicate dsv4-fp4-mi355x-sglang-agentic-hicache entry Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f70ca1a74e..026bed2148 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1551,22 +1551,6 @@ dsv4-fp4-mi355x-sglang-agentic-hicache: - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64] } - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } -dsv4-fp4-mi355x-sglang-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8] } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64] } - - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } - dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 model: deepseek-ai/DeepSeek-V4-Pro From e10fa87e6b6dd9a601c585998516dc7293460d2f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 17 Jul 2026 10:48:55 +0900 Subject: [PATCH 05/35] fix(validation): add run-eval/eval-only fields to agentic matrix entries The eval pipeline injects run-eval and eval-only into agentic entries, but SingleNodeAgenticMatrixEntry and MultiNodeAgenticMatrixEntry had extra='forbid' without those fields, causing validation to reject them. Co-Authored-By: Claude Opus 4.6 --- utils/matrix_logic/validation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..8d9887c81a 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -334,6 +334,8 @@ class MultiNodeAgenticMatrixEntry(BaseModel): exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + run_eval: bool = Field(alias=Fields.RUN_EVAL.value, default=False) + eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @model_validator(mode='after') def validate_worker_hardware_pair(self): From 85f425a5a9b6891edd49db146f3e9f21386c7af8 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sat, 18 Jul 2026 16:02:59 +0900 Subject: [PATCH 06/35] fix(agentic-mtp): tune DPA memory, prefill chunk, and batch sizing Set MEM_FRACTION_STATIC=0.75 under DPA for speculative-decode headroom, double chunked-prefill to 16K/scheduler for agentic prefill tails, set MAX_RUNNING_REQUESTS=CONC, and partition CUDA_GRAPH_MAX_BS across DP ranks. Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 612f2c0f5c..09519b98dc 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -80,6 +80,9 @@ CHUNKED_PREFILL_SIZE=8192 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true + # DPA + MTP needs additional runtime headroom for speculative decode and + # communication buffers beyond SGLang's static KV pool. + MEM_FRACTION_STATIC=0.75 export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true SGLANG_BACKEND_PORT=$((PORT + 1)) SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) @@ -89,9 +92,11 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - CHUNKED_PREFILL_SIZE=$((8192 * TP)) + # SGLang divides the configured chunk across DP schedulers. Use a 16K + # per-scheduler chunk so long agentic prefill tails drain within the + # standard 600-second warmup grace period. + CHUNKED_PREFILL_SIZE=$((16384 * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention @@ -106,8 +111,11 @@ fi # SGLang treats max-running-requests as a global DPA limit and partitions it # internally. CUDA graph capture is per scheduler, so only its batch size is # divided across DP ranks. -MAX_RUNNING_REQUESTS=$((2 * CONC)) +MAX_RUNNING_REQUESTS=$CONC CUDA_GRAPH_MAX_BS=$CONC +if [ "$DP_ATTENTION" = "true" ]; then + CUDA_GRAPH_MAX_BS=$(( (CUDA_GRAPH_MAX_BS + TP - 1) / TP )) +fi [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 # Simulated acceptance-length (AL) settings. @@ -132,10 +140,6 @@ SPEC_ARGS=( --speculative-num-draft-tokens 4 ) -if [ ${#SPEC_ARGS[@]} -gt 0 ]; then - MEM_FRACTION_STATIC=$(awk "BEGIN {printf \"%.2f\", $MEM_FRACTION_STATIC - 0.10}") -fi - SGLANG_CMD=( python3 -m sglang.launch_server --model-path "$MODEL_PATH" From 090cd59d64d6973db0cf9ca04bfcd7c6bee1f66e Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 01:13:42 +0900 Subject: [PATCH 07/35] fix(agentic-mtp): revert mem_fraction_static override, narrow sweep to c48 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disable the 0.75 mem_fraction_static override that caused a 6x KV cache reduction (5.1M → 873K tokens), falling back to the default 0.90. Narrow the sweep to DPA c48 only for a focused retest. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 ++- configs/amd-master.yaml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 09519b98dc..0f9642027c 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -82,7 +82,8 @@ if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true # DPA + MTP needs additional runtime headroom for speculative decode and # communication buffers beyond SGLang's static KV pool. - MEM_FRACTION_STATIC=0.75 + # 0719 + #MEM_FRACTION_STATIC=0.75 export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true SGLANG_BACKEND_PORT=$((PORT + 1)) SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 026bed2148..70215c5ddc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1563,8 +1563,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 52], spec-decoding: mtp } + #- { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } + #- { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 52], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From 441b580caf30698d64e45a09a0b1b9082caf620f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 01:20:32 +0900 Subject: [PATCH 08/35] fix(agentic-mtp): stop dividing CUDA_GRAPH_MAX_BS by TP for DPA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the per-DP-rank division of CUDA_GRAPH_MAX_BS — keep it at CONC so decode batches up to the concurrency level stay on the CUDA graph fast path (was dropping from 52 to 7 with TP8). Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 0f9642027c..3a70ee9833 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -113,10 +113,8 @@ fi # internally. CUDA graph capture is per scheduler, so only its batch size is # divided across DP ranks. MAX_RUNNING_REQUESTS=$CONC +# 0719 CUDA_GRAPH_MAX_BS=$CONC -if [ "$DP_ATTENTION" = "true" ]; then - CUDA_GRAPH_MAX_BS=$(( (CUDA_GRAPH_MAX_BS + TP - 1) / TP )) -fi [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 # Simulated acceptance-length (AL) settings. From e1625509fd958bd33c738a04f823652979a05872 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 01:22:58 +0900 Subject: [PATCH 09/35] fix(agentic-mtp): restore MAX_RUNNING_REQUESTS to 2*CONC Revert MAX_RUNNING_REQUESTS from CONC back to 2*CONC to match the old (good) run configuration that achieved 5,985 completed records. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 3a70ee9833..a84e2a8abf 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -112,7 +112,8 @@ fi # SGLang treats max-running-requests as a global DPA limit and partitions it # internally. CUDA graph capture is per scheduler, so only its batch size is # divided across DP ranks. -MAX_RUNNING_REQUESTS=$CONC +# 0719 +MAX_RUNNING_REQUESTS=$((2 * CONC)) # 0719 CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 From 522b3d38875da637765160c1f6f1cdd41e01970f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 01:24:16 +0900 Subject: [PATCH 10/35] fix(agentic-mtp): reduce chunked prefill back to 8K per scheduler Revert per-scheduler chunked prefill from 16K to 8K to match the old run configuration (8192*TP = 65536 total vs 16384*TP = 131072). Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index a84e2a8abf..9140df9fd3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -97,7 +97,8 @@ if [ "$DP_ATTENTION" = "true" ]; then # SGLang divides the configured chunk across DP schedulers. Use a 16K # per-scheduler chunk so long agentic prefill tails drain within the # standard 600-second warmup grace period. - CHUNKED_PREFILL_SIZE=$((16384 * TP)) + # 0719 + CHUNKED_PREFILL_SIZE=$((8192 * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention From 9d2810a6af181497d3a8d1dec4212eb5683cd90c Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 01:25:51 +0900 Subject: [PATCH 11/35] fix(agentic-mtp): restore GPU_MAX_HW_QUEUES=5 for ROCm overlap Re-add GPU_MAX_HW_QUEUES=5 under DPA to enable compute/memory op overlap on ROCm, matching the old run configuration. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 9140df9fd3..ae67d583e8 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -93,6 +93,8 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 + # 0719 + export GPU_MAX_HW_QUEUES=5 # SGLang divides the configured chunk across DP schedulers. Use a 16K # per-scheduler chunk so long agentic prefill tails drain within the From 7a41734887f604da7917c819565f14227806a135 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 16:12:13 +0900 Subject: [PATCH 12/35] fix(agentic-mtp): clean up script to match ref, expand sweep concurrencies Remove debug comments and 0719 markers, drop commented-out code blocks, and remove duplicate MAX_RUNNING_REQUESTS (already set upstream in script). Expand DPA sweep to full concurrency range [4,8,16,32,48,52]. Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 16 ---------------- configs/amd-master.yaml | 4 +--- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index ae67d583e8..72753f66e5 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -80,10 +80,6 @@ CHUNKED_PREFILL_SIZE=8192 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true - # DPA + MTP needs additional runtime headroom for speculative decode and - # communication buffers beyond SGLang's static KV pool. - # 0719 - #MEM_FRACTION_STATIC=0.75 export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true SGLANG_BACKEND_PORT=$((PORT + 1)) SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) @@ -93,13 +89,8 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 - # 0719 export GPU_MAX_HW_QUEUES=5 - # SGLang divides the configured chunk across DP schedulers. Use a 16K - # per-scheduler chunk so long agentic prefill tails drain within the - # standard 600-second warmup grace period. - # 0719 CHUNKED_PREFILL_SIZE=$((8192 * TP)) PARALLEL_ARGS+=( --dp "$TP" @@ -111,13 +102,6 @@ fi if [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") fi - -# SGLang treats max-running-requests as a global DPA limit and partitions it -# internally. CUDA graph capture is per scheduler, so only its batch size is -# divided across DP ranks. -# 0719 -MAX_RUNNING_REQUESTS=$((2 * CONC)) -# 0719 CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 70215c5ddc..34a827ffe5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1563,9 +1563,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - #- { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } - #- { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 52], spec-decoding: mtp } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48, 52], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From 15a8fea7946f06644dabe7d63f834782a0366502 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 16:29:02 +0900 Subject: [PATCH 13/35] fix(validation): remove run_eval/eval_only from MultiNodeAgenticMatrixEntry Main branch already handles these fields via Optional on SingleNodeAgenticMatrixEntry; MultiNodeAgenticMatrixEntry doesn't need them. Co-Authored-By: Claude Opus 4.6 --- utils/matrix_logic/validation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 8d9887c81a..9f19572f08 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -334,8 +334,6 @@ class MultiNodeAgenticMatrixEntry(BaseModel): exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) - run_eval: bool = Field(alias=Fields.RUN_EVAL.value, default=False) - eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @model_validator(mode='after') def validate_worker_hardware_pair(self): From 7d1713f70e6b8bb74c5a9b8c79aee07ac6b42e11 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 16:33:58 +0900 Subject: [PATCH 14/35] fix(agentic-mtp): add MAX_RUNNING_REQUESTS=2*CONC with comment Set MAX_RUNNING_REQUESTS to 2*CONC to allow subagent fan-out to exceed the concurrency target without clipping request bursts. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 72753f66e5..a00789622e 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -102,9 +102,11 @@ fi if [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") fi +# AgentX concurrency counts live session trees, not individual requests. +# Allow subagent fan-out to exceed CONC without clipping request bursts. +MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 - # Simulated acceptance-length (AL) settings. export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=high From a68e8bd4859bbddff2cd8e237fb2d0459f60592e Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 17:28:47 +0900 Subject: [PATCH 15/35] fix(agentic-mtp): drop c52 from sweep to avoid GPU OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove concurrency 52 from the DPA MTP sweep — at c52 with max_running_requests=104 and mem_fraction_static=0.9, the eval-only run exhausts GPU memory (0 MB free) during speculative decode prefills, causing HSA_STATUS_ERROR_OUT_OF_RESOURCES. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 34a827ffe5..b2e085bf24 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1563,7 +1563,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48, 52], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From c9d41c17686feb592d66390a82b396a7442a37ac Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 19 Jul 2026 19:59:56 +0900 Subject: [PATCH 16/35] feat(agentic-mtp): add cache metrics capture and eval-only support Add capture_cache_metrics() to snapshot SGLang cache/usage metrics before and after benchmark runs for debugging. Add eval-only mode branch to run_eval instead of replay when EVAL_ONLY=true. Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index a00789622e..c010465550 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -169,6 +169,16 @@ echo "Starting SGLang server for MI355X..." SERVER_PID=$! echo "Server PID: $SERVER_PID" +capture_cache_metrics() { + { + echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" + curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ + | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ + || true + echo "============================================================" + } >> "$SERVER_LOG" +} + wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" if [ "$USE_SGLANG_ROUTER" = "true" ]; then @@ -190,9 +200,17 @@ if [ "$USE_SGLANG_ROUTER" = "true" ]; then echo "Router PID: $ROUTER_PID" wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" fi - # ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" -REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" -run_agentic_replay_and_write_outputs "$RESULT_DIR" +if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then + capture_cache_metrics + trap capture_cache_metrics EXIT +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi From 9facbfe7990c9d7998e458ec79566b1e9f0dd9b6 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 20 Jul 2026 10:28:57 -0700 Subject: [PATCH 17/35] fix(agentic-mtp): disable default thinking to avoid context overflow Disable SGLANG_DEFAULT_THINKING=1 which causes DSv4's blocks to accumulate across SWE-bench agent steps, snowballing to 3.6M tokens and exceeding the 1M context limit. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index c010465550..9b39c55d7d 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -108,7 +108,8 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 # Simulated acceptance-length (AL) settings. -export SGLANG_DEFAULT_THINKING=1 +# openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': "The input (3620936 tokens) is longer than the model's context length (1048576 tokens).", 'type': 'BadRequestError', 'param': None, 'code': 400} +#export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_SIMULATE_ACC_LEN=2.49 export SGLANG_SIMULATE_ACC_METHOD=match-expected From 37294112eec62491e24b8509363854bc99e3531e Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 20 Jul 2026 11:04:37 -0700 Subject: [PATCH 18/35] fix(agentic-mtp): also disable reasoning_effort=high Disable SGLANG_DSV4_REASONING_EFFORT=high alongside thinking to further reduce context accumulation during SWE-bench eval. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 9b39c55d7d..ca9fbcc33c 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -110,7 +110,7 @@ CUDA_GRAPH_MAX_BS=$CONC # Simulated acceptance-length (AL) settings. # openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': "The input (3620936 tokens) is longer than the model's context length (1048576 tokens).", 'type': 'BadRequestError', 'param': None, 'code': 400} #export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=high +#export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_SIMULATE_ACC_LEN=2.49 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token From cc5888d9bf4053be61e4d0ab2c167ae64d573732 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 20 Jul 2026 18:56:30 -0700 Subject: [PATCH 19/35] fix(agentic-mtp): re-enable thinking mode for SWE-bench quality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-enable SGLANG_DEFAULT_THINKING=1 and SGLANG_DSV4_REASONING_EFFORT=high. Without thinking, DSv4 produces 0% SWE-bench resolve rate — patches break target projects. Context overflow from thinking accumulation needs to be solved at the agent level (context truncation), not by disabling the model's reasoning capability. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index ca9fbcc33c..e75558db0e 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -109,8 +109,8 @@ CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 # Simulated acceptance-length (AL) settings. # openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': "The input (3620936 tokens) is longer than the model's context length (1048576 tokens).", 'type': 'BadRequestError', 'param': None, 'code': 400} -#export SGLANG_DEFAULT_THINKING=1 -#export SGLANG_DSV4_REASONING_EFFORT=high +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_SIMULATE_ACC_LEN=2.49 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token From f4293c66a487c08183249eace16f9b39eaf7d6fc Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 20 Jul 2026 19:06:44 -0700 Subject: [PATCH 20/35] fix(agentic-mtp): replace prefill-delayer with dp-attention-local-control-broadcast Switch from --enable-prefill-delayer to --enable-dp-attention-local-control-broadcast for DPA mode. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index e75558db0e..9d8dd2610d 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -95,7 +95,7 @@ if [ "$DP_ATTENTION" = "true" ]; then PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention - --enable-prefill-delayer + --enable-dp-attention-local-control-broadcast ) fi From ad0e95dfd92f34be736208523bd5ce3e382b3741 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 21 Jul 2026 14:38:31 -0700 Subject: [PATCH 21/35] exclude mia1-p01-g09,mia1-p01-g11 from MI355X salloc Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- runners/launch_mi355x-amds.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c870fd87e0..b2e085bf24 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1563,7 +1563,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 32, 48], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 75a67c1c58..4cdc314a0a 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -268,7 +268,7 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" set -x - salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" + salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" --exclude=mia1-p01-g09,mia1-p01-g11 JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" From 79eafdad822bd08e791dcf4a7f9075f51c7e8c52 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 22 Jul 2026 11:18:27 -0700 Subject: [PATCH 22/35] [AgentX] DSv4 FP4 MI355X SGLang MTP agentic: drop conc-48 from sweep conc-48 causes KV cache pressure on nodes with less available GPU memory, leading to inconsistent benchmark results. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 56bea05a18..c551a18dc8 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1580,7 +1580,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From 2fb47c837afb9deff590ab468f506b11ea8f3b9a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 19:52:57 +0900 Subject: [PATCH 23/35] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic: force lm-eval eval framework Export EVAL_FRAMEWORK=lm-eval so run_eval never falls back to the agentic swebench default (EVAL_FRAMEWORK takes precedence over scenario_default). Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh index 241f042ff2..491fda060f 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh @@ -29,6 +29,13 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. +export EVAL_FRAMEWORK="lm-eval" + check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION if [[ -n "${SLURM_JOB_ID:-}" ]]; then From cf9b71c71cca13258d3a4f6f834963ca24edeebb Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 19:59:00 +0900 Subject: [PATCH 24/35] [AMD] [AGENTX] dsv4 mi355x: force lm-eval on sglang-mtp recipe; drop runner node exclude - dsv4_fp4_mi355x_sglang_mtp.sh: export EVAL_FRAMEWORK=lm-eval (never swebench). - dsv4_fp4_mi355x_vllm.sh: revert the EVAL_FRAMEWORK line (moved to sglang-mtp). - launch_mi355x-amds.sh: remove --exclude=mia1-p01-g09,mia1-p01-g11 from salloc. Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 7 +++++++ benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh | 7 ------- runners/launch_mi355x-amds.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 9d8dd2610d..09e1308268 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -14,6 +14,13 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. +export EVAL_FRAMEWORK="lm-eval" + check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION if [[ -n "$SLURM_JOB_ID" ]]; then diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh index 491fda060f..241f042ff2 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh @@ -29,13 +29,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Force the eval framework to lm-eval for this recipe. run_eval derives its -# default as swebench for agentic scenarios (scenario_default=swebench when -# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence -# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so -# setting it here makes the effective framework always lm-eval, never swebench. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION if [[ -n "${SLURM_JOB_ID:-}" ]]; then diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 4cdc314a0a..75a67c1c58 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -268,7 +268,7 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" set -x - salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" --exclude=mia1-p01-g09,mia1-p01-g11 + salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" From efac275f1d471dfc41e2fa865739b62d51013432 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 00:17:16 +0900 Subject: [PATCH 25/35] [AMD] [AGENTX] dsv4-fp4-mi355x-sglang-mtp: gate simulated acceptance to throughput runs Only export SGLANG_SIMULATE_ACC_* when EVAL_ONLY=false. Simulated acceptance bypasses real target verification, so it must be off for accuracy (EVAL_ONLY) runs or it corrupts the eval score. Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 09e1308268..fdba23d634 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -118,9 +118,12 @@ CUDA_GRAPH_MAX_BS=$CONC # openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': "The input (3620936 tokens) is longer than the model's context length (1048576 tokens).", 'type': 'BadRequestError', 'param': None, 'code': 400} export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=high -export SGLANG_SIMULATE_ACC_LEN=2.49 -export SGLANG_SIMULATE_ACC_METHOD=match-expected -export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token + +if [ "${EVAL_ONLY:-false}" = "false" ]; then + export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton From 9b3912e645cbf74b2c9088308e5fa2d80e68126f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 08:39:15 +0900 Subject: [PATCH 26/35] [AMD] [AGENTX] dsv4-fp4-mi355x-sglang-agentic-mtp: add TP8 dp-off conc 1,2 arm Add a pure-TP (dp-attn false) low-concurrency arm (conc 1,2, spec-decoding mtp) alongside the existing DEP conc 4-32 arm. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 849feba75a..ae6ef781cb 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1695,6 +1695,7 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: + - { tp: 8, dp-attn: false, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: From a11a3774a73f53c44287ab73976aabfba952d5f0 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 09:43:56 +0900 Subject: [PATCH 27/35] [AMD] [AGENTX] dsv4-fp4-mi355x-sglang-agentic-mtp: drop TP8 dp-off conc 1,2 arm Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ae6ef781cb..849feba75a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1695,7 +1695,6 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: false, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: From 10a4f03689cce04549059aaae79e74bfd5f5f5d3 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 09:28:16 +0900 Subject: [PATCH 28/35] [AMD][AgentX] dsv4-fp4-mi355x-sglang-agentic-mtp: reduce mem_fraction_static to 0.85 and chunked_prefill_size to 4096 to avoid runtime OOM Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index fdba23d634..9b6557fa53 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -82,8 +82,8 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=1000000 USE_SGLANG_ROUTER=false SGLANG_BACKEND_PORT="$PORT" ROUTER_LOG="$RESULT_DIR/router.log" -MEM_FRACTION_STATIC=0.90 -CHUNKED_PREFILL_SIZE=8192 +MEM_FRACTION_STATIC=0.85 +CHUNKED_PREFILL_SIZE=4096 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true @@ -98,7 +98,7 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_USE_REDUCE_SCATTER=1 export GPU_MAX_HW_QUEUES=5 - CHUNKED_PREFILL_SIZE=$((8192 * TP)) + CHUNKED_PREFILL_SIZE=$((4096 * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention From 43d99c5ad1276dc89067ee1828c2d1a82d5c53c6 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 11:30:36 +0900 Subject: [PATCH 29/35] [AMD][AgentX] dsv4-fp4-mi355x-sglang-agentic-mtp: update conc-list to [48,64] and add hicache dram arm Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d45886a2ea..fff15a73ba 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1695,7 +1695,8 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From 32a51532eac13478742fecb80b658aff1fee8095 Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Wed, 29 Jul 2026 21:48:57 -0500 Subject: [PATCH 30/35] fix(agentx): pin additive AIPerf main warmup (#2415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the shared AIPerf submodule to the synchronized main revision with additive per-lane warmup after mandatory primers while retaining timing-faithful spread phase starts. Align the workflow documentation with the request-count fast preset. 中文：将共享 AIPerf 子模块固定到已同步的 main 版本，在必需预热请求之后执行额外的逐 lane 预热，并保留符合轨迹时序的分散阶段启动。同步更新工作流文档，使其准确描述基于请求数量的快速预设。 --- .github/workflows/README.md | 11 ++++--- .../workflows/benchmark-multinode-tmpl.yml | 2 +- .github/workflows/benchmark-tmpl.yml | 2 +- .github/workflows/e2e-tests.yml | 4 +-- .gitmodules | 1 - AGENTS.md | 4 +-- benchmarks/benchmark_lib.sh | 33 ++++++++++--------- .../multi_node/amd_utils/server_sglang.sh | 2 +- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 1 - .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 1 - ...gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml | 1 - ...gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml | 1 - utils/aiperf | 2 +- utils/process_changelog.py | 23 +++++++++++-- utils/test_process_changelog.py | 28 ++++++++++++++++ 15 files changed, 80 insertions(+), 36 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0c1454ab61..5ab5160d76 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -159,11 +159,12 @@ are also reusable. ## AgentX Fast Mode -Add `agentx-fast` alongside one primary sweep label to run the 5-minute cache -warmup and 20-minute profiling preset for single- and multi-node AgentX -throughput jobs. Fixed-sequence throughput and eval jobs retain their canonical -settings. Adding or removing the modifier restarts the active sweep. Fast-mode -runs are not eligible for artifact reuse after merge. +Add `agentx-fast` alongside one primary sweep label to run one additional +warmup request per AgentX lane after mandatory primers and a 20-minute profile +for single- and multi-node AgentX throughput jobs. Fixed-sequence throughput +and eval jobs retain their canonical settings. Adding or removing the modifier +restarts the active sweep. Fast-mode runs are not eligible for artifact reuse +after merge. ## Reusing an Approved PR Full Sweep diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 3f46ddb353..10411ad069 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -155,7 +155,7 @@ on: required: false default: "3600" agentx-fast: - description: "Use the AgentX 5-minute warmup and 20-minute profiling preset" + description: "Use one warmup request per AgentX lane and a 20-minute profile" type: boolean required: false default: false diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b86ed96705..46a80c8743 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -130,7 +130,7 @@ on: type: string default: '3600' agentx-fast: - description: "Use the AgentX 5-minute warmup and 20-minute profiling preset" + description: "Use one warmup request per AgentX lane and a 20-minute profile" required: false type: boolean default: false diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 0420cf73dc..34fba860f4 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -22,7 +22,7 @@ on: type: string default: "" agentx-fast: - description: "AgentX fast feedback: 5-minute cache warmup and 20-minute profile." + description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." required: false type: boolean default: false @@ -56,7 +56,7 @@ on: type: string default: "" agentx-fast: - description: "AgentX fast feedback: 5-minute cache warmup and 20-minute profile." + description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." required: false type: boolean default: false diff --git a/.gitmodules b/.gitmodules index b026356de5..79fbb076aa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "utils/aiperf"] path = utils/aiperf url = https://github.com/SemiAnalysisAI/aiperf.git - branch = cquil11/aiperf-agentx-v1.0 diff --git a/AGENTS.md b/AGENTS.md index f8dfbf9508..9e93f49f6d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ PRs do not run the sweep automatically - `run-sweep.yml` is gated on a primary s - `full-sweep-fail-fast` - runs the full intermediate concurrency sweep behind the same sequential single-node canary gate as `full-sweep-enabled` (so a globally broken change burns one job, not the whole fan-out), and with `strategy.fail-fast` enabled on every matrix: the first failure in a matrix cancels that matrix's remaining jobs. Fail-fast is matrix-scoped, so the other matrices (1k1k vs 8k1k vs agentic vs evals) keep running and self-terminate on their own first failure; their completed results remain valid. The failing job keeps its red *failure* conclusion and the run concludes failed. **This is the strongly recommended default for full sweeps** (image bumps, recipe changes, bring-up) - a failure means the rest of that matrix is wasted GPU time. Caveat: one flaky job kills its matrix's in-flight results; if that repeatedly bites a specific config, fall back to `full-sweep-enabled` for that PR. - `full-sweep-fail-fast-no-canary` - same as `full-sweep-fail-fast` but without the canary gate: all matrices fan out immediately. Use when the canary is flaky or not representative of the affected configuration but you still want per-matrix fail-fast. -`all-evals`, `evals-only`, and `agentx-fast` are optional modifier labels. Combine them with one primary sweep label. `all-evals` expands eval selection to every generated fixed-sequence configuration without changing throughput. `evals-only` suppresses throughput while keeping the default eval subset; combining both eval modifiers runs every eval and no throughput. `agentx-fast` applies the 5-minute warmup and 20-minute profiling preset to single- and multi-node AgentX throughput jobs only; fixed-sequence and eval jobs are unchanged. `all-evals` remains eligible for artifact reuse when paired with an eligible full-sweep label. Runs with `evals-only` or `agentx-fast` are not eligible for artifact reuse. +`all-evals`, `evals-only`, and `agentx-fast` are optional modifier labels. Combine them with one primary sweep label. `all-evals` expands eval selection to every generated fixed-sequence configuration without changing throughput. `evals-only` suppresses throughput while keeping the default eval subset; combining both eval modifiers runs every eval and no throughput. `agentx-fast` reduces deterministic warmup to one request per lane and profiling to 20 minutes for single- and multi-node AgentX throughput jobs; fixed-sequence and eval jobs are unchanged. `all-evals` remains eligible for artifact reuse when paired with an eligible full-sweep label. Runs with `evals-only` or `agentx-fast` are not eligible for artifact reuse. **The sweep does not trigger while the PR has merge conflicts.** Even with a sweep label applied, the `run-sweep.yml` workflow will not start until the PR cleanly merges into main — a stale claude/* or update-* branch with a `perf-changelog.yaml` conflict (the common case) will sit in NO_SWEEP / NO_SUCCESS until rebased. Resolution recipe is documented in `KLAUD_DEBUG.md §1.1`: `git merge origin/main`, then `git checkout origin/main -- perf-changelog.yaml`, then re-append the PR's own changelog entry at the tail. Don't 3-way merge `perf-changelog.yaml`; whitespace edits silently re-trigger the deletion check. @@ -119,7 +119,7 @@ gh api -X POST \ Inputs: top-level `ref` (required) is the workflow ref to dispatch from, almost always `main`. `inputs[ref]` is the repo ref under test (defaults to the dispatch ref's `github.sha`). `inputs[generate-cli-command]` (required) is passed verbatim to `generate_sweep_configs.py` - test locally first. `inputs[test-name]` is the display name in the Actions UI. `inputs[duration-override]` overrides per-config duration (seconds); empty = use matrix value. -For an AgentX preflight, add `-F 'inputs[agentx-fast]=true'` to the dispatch command. This selects a 5-minute cache warmup and 20-minute profile for every AgentX job in that e2e dispatch and takes precedence over `duration-override`. Use it to get faster signal while debugging, then run the official sweep without `agentx-fast`; canonical AgentX timing remains a 10-minute cache warmup and 1-hour profile. +For an AgentX preflight, add `-F 'inputs[agentx-fast]=true'` to the dispatch command. This reduces deterministic warmup to one request per lane and profiling to 20 minutes for every AgentX job. Use it to get faster feedback while debugging, then run the official sweep without `agentx-fast`; canonical AgentX warmup remains 10 requests per lane with a 1-hour profile. The POST returns no body and no run ID - find the run with `gh run list` below. diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 7895427bc2..92d0c2eac4 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1736,22 +1736,19 @@ build_replay_cmd() { # the worker threads the server's live assistant response back into the # session. # - # The scenario plugin locks: --cache-bust first_turn_prefix and - # --trace-idle-gap-cap-seconds 10 (per-trace idle-gap compression - # against parent + subagent request-start timestamps; supersedes the - # legacy --use-think-time-only / --inter-turn-delay-cap-seconds path), - # and auto-injects them — so we do not pass them. See - # utils/aiperf/docs/tutorials/agentx-mvp.md. + # The scenario plugin locks --cache-bust first_turn_prefix and a 10-second + # whole-system idle cap. Source end-to-start delays remain intact; the cap + # shifts all pending timers uniformly only when no request is active or + # ready. See utils/aiperf/docs/tutorials/agentx-mvp.md. local result_dir="$1" local duration="$DURATION" - local cache_warmup_duration="${AIPERF_AGENTIC_CACHE_WARMUP_DURATION:-600}" + local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}" - # Fast mode is an e2e-only feedback preset used before canonical one-hour - # sweeps. AIPerf already exposes both controls, so no AIPerf patch is - # required. + # Fast mode minimizes setup by advancing each trajectory lane only once + # and shortens profiling to 20 minutes. if [[ "${AIPERF_EXPERIMENTAL_FAST:-0}" == "1" ]]; then duration=1200 - cache_warmup_duration=300 + warmup_requests_per_lane=1 fi export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES:-0}" @@ -1765,6 +1762,9 @@ build_replay_cmd() { # aiperf validates that SERVICE_PROFILE_CONFIGURE_TIMEOUT >= # DATASET_CONFIGURATION_TIMEOUT at startup. Bump it in lockstep. export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800 + # Headless realtime metrics are opt-in on current AIPerf main. Enable the + # rolling TTFT/ITL/throughput block and emit it every 30 seconds. + export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" REPLAY_CMD+=" --url http://localhost:$PORT" REPLAY_CMD+=" --endpoint /v1/chat/completions" @@ -1773,6 +1773,7 @@ build_replay_cmd() { REPLAY_CMD+=" --model $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" + REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" # Fail runs once more than 10% of requests error. This keeps known # transient low-rate failures from killing long sweeps while still @@ -1784,10 +1785,12 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" - # After the normal t* snapshot warmup, continue those exact trajectories - # with one-token outputs and no idle delays. Profiling begins only after - # those requests drain and resumes from the resulting live state. - REPLAY_CMD+=" --agentic-cache-warmup-duration $cache_warmup_duration" + # After the normal t* snapshot primers, advance every trajectory lane by + # this many additional one-token requests with no idle delay. Profiling + # begins after those requests drain and resumes from the resulting live + # state. Do not pass --burst-phase-starts: AIPerf main's spread default + # preserves each lane's recorded phase-start offset. + REPLAY_CMD+=" --warmup-requests-per-lane $warmup_requests_per_lane" # Give long-context warmup requests up to 30 minutes to drain before # declaring warmup failed. Recipes whose saturation arms carry a larger # in-flight working set may override via AGENTIC_WARMUP_GRACE_PERIOD diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 4e917fd26e..84c9e598c0 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -986,7 +986,7 @@ if [ "$NODE_RANK" -eq 0 ]; then DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ - AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ + AIPERF_WARMUP_REQUESTS_PER_LANE AIPERF_EXPERIMENTAL_FAST AIPERF_UNSAFE_OVERRIDE \ AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do if [[ -n "${!_v+x}" ]]; then diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 6d1681793b..192e0f5b95 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -141,7 +141,6 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 06b916659f..6a950787fe 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -144,7 +144,6 @@ benchmark: # the zero decode-worker count instead of duplicating TP into P and D. IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml index cc706a4aac..dd58dde28f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml @@ -198,7 +198,6 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml index fa856fe218..00a876ebe8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml @@ -199,7 +199,6 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/utils/aiperf b/utils/aiperf index 381758a88f..be758d6218 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 381758a88f5fa6335455ef5b0ded9557c1162814 +Subproject commit be758d6218268171e2957fbec9d4f557275bca2d diff --git a/utils/process_changelog.py b/utils/process_changelog.py index c4b226ada0..bc37d492c5 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -16,6 +16,17 @@ SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding") +def _freeze_config_value(value): + """Convert JSON-shaped config values into deterministic hashable values.""" + if isinstance(value, dict): + return tuple( + sorted((key, _freeze_config_value(item)) for key, item in value.items()) + ) + if isinstance(value, list): + return tuple(_freeze_config_value(item) for item in value) + return value + + def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: result = subprocess.run( ["git", "diff", base_ref, head_ref, "--", filepath], @@ -54,8 +65,8 @@ def trim_conc(entries: list[dict]) -> list[dict]: the source ordering of ``conc-list`` / ``conc-start``. Input comes from ``json.loads(subprocess.stdout)`` so ``conc`` is always - ``int`` (single-node) or ``list`` (multi-node); other single-node fields - are hashable scalars. + ``int`` (single-node) or ``list`` (multi-node). Other fields may contain + nested dictionaries or lists, such as KV-offload backend metadata. - Single-node entries: group by every other field and keep only the entry with the lowest ``conc`` per group. @@ -72,7 +83,13 @@ def trim_conc(entries: list[dict]) -> list[dict]: out.append(entry) continue - key = tuple(sorted((k, v) for k, v in entry.items() if k != "conc")) + key = tuple( + sorted( + (k, _freeze_config_value(v)) + for k, v in entry.items() + if k != "conc" + ) + ) groups.setdefault(key, []).append(len(out)) out.append(entry) diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 476bec0742..0665ce9ecd 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -15,6 +15,34 @@ def _scenario_values(command): return command[index:] +def test_trim_conc_supports_nested_backend_metadata(): + common = { + "model": "moonshotai/Kimi-K3", + "kv-offloading": "dram", + "kv-offload-backend": { + "name": "vllm-simple", + "settings": {"tiers": ["cpu", "gpu"]}, + }, + } + entries = [ + {**common, "conc": 8}, + {**common, "conc": 2}, + { + **common, + "kv-offload-backend": {"name": "lmcache"}, + "conc": 4, + }, + ] + + trimmed = process_changelog.trim_conc(entries) + + assert [entry["conc"] for entry in trimmed] == [2, 4] + assert [entry["kv-offload-backend"]["name"] for entry in trimmed] == [ + "vllm-simple", + "lmcache", + ] + + def test_config_key_expansion_is_deterministic_and_deduplicated(): master_config = { "config-b": {}, From e253ac817c27d56aeabc5e49504c3d62322ae8c0 Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:37:53 -0700 Subject: [PATCH 31/35] fix: recover PR 2380 ingest via sweep reuse (#2418) * fix(kimik3): keep only TRTLLM_RAGGED in the DSpark recipe, revert the rest Reduce the flag alignment to its one surviving piece and rebuild on main. The full upstream Blackwell/NVIDIA block killed the engine core on every completed concurrency -- c1/c2/c4/c8 across both KV arms in run 30378755933 -- roughly 13-16 min in, during agentic warmup: vllm/model_executor/layers/fused_moe/runner/shared_experts.py:165 assert self._output[self._output_idx] is None AssertionError (all 8 TP ranks simultaneously) K3 declares num_shared_experts 2 and this image resolves its MoE backend to FLASHINFER_TRTLLM_MXFP4_MXFP8 / TrtLlmMxfp4ExpertsMonolithic, so that reads as an image-side invariant violation rather than a bad configuration value. The same crash hit the non-DSpark recipe, so it is not spec-decoding specific. Retained: --attention-config mla_prefill_backend=TRTLLM_RAGGED Reverted: gpu-memory-utilization 0.95 -> 0.90 max-num-seqs 32 -> 2*CONC max-model-len 1000000 -> 1048576 max-num-batched-tokens 32768 -> dropped (8192 default) --no-enable-flashinfer-autotune -> dropped VLLM_USE_V2_MODEL_RUNNER=1 -> dropped (V2 is on by default anyway) cudagraph capture consequently enumerates 2*CONC token-multiples of (1+7)=8 again, instead of a fixed 32-entry ladder to 256. Rebased onto main rather than amending, since main had advanced four commits and the changelog diff had started reading main's tail as deletions. * fix(kimik3): enable VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION on the DSpark recipe The upstream recipe sets this on BOTH its blackwell and nvidia paths (recipes.vllm.ai/moonshotai/Kimi-K3), and this repo has never set it. It defaults to 0. It matters because of where it lands: vllm/models/kimi_k3/nvidia/model.py:549 threads it into LatentMoERunner as runner_args={"enable_k3_latent_moe_tail_fusion": ...}, and that is the same runner whose shared-experts output buffer asserted when the wider flag alignment was tried: vllm/model_executor/layers/fused_moe/runner/shared_experts.py:165 assert self._output[self._output_idx] is None (all 8 TP ranks) So with it unset we have been running a MoE tail path upstream never exercises, on every K3 run here including the green ones. That is a better explanation for the crash than any of the six flags reverted earlier. Enabled on its own so its effect is isolated. If this holds, the remaining upstream deltas can be re-added in order: gpu-memory-utilization 0.95, max-num-seqs 32 (spec arm), --no-enable-flashinfer-autotune, VLLM_USE_V2_MODEL_RUNNER=1. Not re-adding --max-num-batched-tokens 32768: re-reading the recipe YAML shows that value appears nowhere for blackwell (AMD is 4096, PD prefill 16384, PD decode 32), so it was introduced in error. --max-model-len is likewise 1048576 upstream, not 1000000, and already correct here. * feat(kimik3): DSpark at level 2 with synthetic acceptance pinned to golden AL Adopt the AgentX-prescribed measurement methodology for the DSpark arm, per golden_al_distribution/README.md: "a submission may choose any supported draft length, but it may not substitute a different acceptance target", because "InferenceX is evaluating inference-system performance, not the ability to fine-tune a benchmark-specific speculative head". The README also states up front that AL is workload-dependent, so pinning the committed acceptance is what makes submissions comparable -- running the acceptance this corpus happens to produce would be the non-conforming choice. throughput: rejection_sample_method synthetic, synthetic_acceptance_length 2.51 (committed golden AL at K=2, probabilistic curve) EVAL_ONLY: rejection_sample_method block (real verification) The eval split matters: synthetic acceptance commits drafts regardless of target logits, so text is wrong and SWE-bench scores 0.0000. This follows dsv4_fp4_b300_vllm_mtp.sh; kimik2.5_fp4_b300_mtp.sh omits the split. Level 2, not 7. Draft length IS ours to choose, and verify width is a real system cost synthetic acceptance does not remove -- at K=7 the target verifies 8 positions per step, which loses to no-speculation once the GPU saturates even at the forced golden AL of 3.84. With cost ~ 1 + m(K-1) + f: K=2 clears break-even across the m range (~1.7x low conc, ~1.2x saturated), K=3 is a wash at saturation (~0.95x), K=7 is negative (~0.54x). cudagraph capture therefore enumerates multiples of (1+2)=3. Recorded in the script and config headers, because the throughput numbers will otherwise be misread: the draft's REAL acceptance on this corpus is 1.16-2.01 with 13-41% position-1 acceptance. Its native window is 32k (rope yarn factor 32 over original_max_position_embeddings 32768, stretched to the declared 1M), and the identical image, draft and spec config reach AL 4.18 / 0.826 position-1 on short-context work. The synthetic numbers measure the system at a prescribed acceptance; they are not evidence the draft predicts well at 100k+ context. * chore: prepare PR 2380 ingest recovery * fix: recover PR 2380 ingest --- perf-changelog.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 938fb5f402..41773164b6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5301,3 +5301,18 @@ - "Enable multiple frontends and MessagePack request-plane encoding, and reduce 1P4D decode GPU memory utilization to 0.85" - "Drop the recipe setup_script entirely (removes the DeepGEMM NVLink barrier timeout patch); required container deps now ship in vLLM 0.25.1, on NVIDIA/srt-slurm v1.0.31" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2139 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + description: + - "Add the DSpark speculative-decoding variant of kimik3-fp4-b300-vllm-agentic: Kimi-K3 (MXFP4, 2.8T MoE, KDA/MLA hybrid, 1M native context) on B300 with vLLM, agentic-coding scenario only -- no fixed-seq-len (1k1k / 8k1k) arms." + - "Speculative config: method dspark on the Inferact/Kimi-K3-DSpark draft head, num_speculative_tokens 2, attention_backend FLASHINFER_MLA, draft_sample_method probabilistic, with rejection_sample_method synthetic and synthetic_acceptance_length 2.51 -- the committed golden AL at K=2 on the probabilistic curve (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml). This follows the AgentX policy in golden_al_distribution/README.md: a submission may choose any supported draft length but may not substitute a different acceptance target, because InferenceX measures inference-system performance rather than the fitness of a benchmark-specific speculative head. The greedy/standard curve gives 2.45 at K=2; the curves are not mixed." + - "EVAL_ONLY runs switch to real block verification. Synthetic acceptance commits drafted tokens regardless of the target logits, so generated text is wrong and SWE-bench scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes, and which kimik2.5_fp4_b300_mtp.sh omits; dsv4 is the pattern followed here." + - "Level 2 rather than 7. Draft length is the submission's choice and verify width is a real system cost synthetic acceptance does not remove: at K=7 the target verifies 8 positions per step, which loses to no-speculation once the GPU saturates even at the forced golden AL of 3.84 (K=2 clears break-even at both ends of the cost range; K=3 is a wash at saturation; K=7 is negative). cudagraph capture consequently enumerates multiples of (1+2)=3, since vLLM rounds configured sizes to multiples of (1+N) and would otherwise cover only MAX_NUM_SEQS/3 sequences. The drafter's KV footprint costs ~25% of GPU KV regardless of K and remains a genuine cost of enabling spec decoding on 100k+ prompts." + - "The Inferact/Kimi-K3-DSpark draft is not pre-staged on the B300 cluster, so it is downloaded once to the launcher's writable models dir (/data/models, same placement as DeepSeek-V4-Pro-DSpark) and cached across the sweep. It must not go next to the target: Kimi-K3 is in STAGED_MODELS, so dirname(MODEL_PATH) is the read-only /scratch/models mount." + - "Config key uses the -dspark suffix rather than the repo-wide -mtp convention, to name the actual speculative method. Note the schema field stays spec-decoding: mtp (its Literal has no dspark member), so generated exp-names, artifacts and job labels still read _spec-mtp, and the benchmark script remains kimik3_fp4_b300_vllm_mtp.sh because the launcher derives its _mtp suffix from that field rather than from the key." + - "Serve flags stay on the values that ran 12/12 green in run 30326393603, with one exception adopted from recipes.vllm.ai/moonshotai/Kimi-K3: --attention-config mla_prefill_backend=TRTLLM_RAGGED (the non-DSpark sibling gets the same swap in #2385). Adopting the rest of the upstream Blackwell/NVIDIA block at the same time -- gpu-memory-utilization 0.95, max-num-seqs 32, max-model-len 1000000, max-num-batched-tokens 32768, --no-enable-flashinfer-autotune, VLLM_USE_V2_MODEL_RUNNER=1 -- killed the engine core on every completed concurrency (c1/c2/c4/c8, both KV arms, run 30378755933) about 13-16 min in during agentic warmup, with an assertion in the FlashInfer MoE runner's shared-experts output buffer (vllm/model_executor/layers/fused_moe/runner/shared_experts.py:165, assert self._output[self._output_idx] is None) firing on all 8 TP ranks at once. So max-num-seqs stays at 2*CONC and cudagraph capture enumerates 2*CONC token-multiples of (1+7)=8 rather than a fixed 32/256 ladder." + - "Measured acceptance does NOT reproduce the golden curve on this workload. A partial run (30337911387, 8 of 24 jobs before an unrelated enroot/pyxis flake cancelled the rest) recorded mean acceptance length 1.18-1.68 against the golden 3.84, per-position acceptance decaying 0.26 / 0.12 / 0.087 / 0.056 / 0.037 / 0.019 / 0.007 and an 8-10% average draft acceptance rate -- drafting 489-515 tok/s to accept 41-48. Throughput regressed 20-63% vs the non-MTP arms at conc 4 and 8. The spec config itself matches upstream exactly, and the draft supports the full 1M context (max_position_embeddings 1048576), so the gap is workload: the golden curve was collected on SPEED-Bench coding at max_model_len 16384 with short single-turn prompts, whereas agentic replays 118-230k-token multi-turn tool-calling traces. Treat the SPEED-Bench curve as non-transferable here and re-measure AL on the agentic corpus before trusting a level-7 configuration. Note rejection_sample_method=block is what exposed this: a synthetic_acceptance_length pinned to 3.84 would have reported a phantom speedup." + - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." + - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418 From 7de0ce2a10af62573b9c48d24d02b9d1b560cb4f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 14:55:56 +0900 Subject: [PATCH 32/35] [AMD][AgentX] dsv4-fp4-mi355x-sglang-agentic-mtp: bump sglang image to v0.5.16-20260729 and set kvnone conc-list to [72] Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index fff15a73ba..488b9c5fea 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1684,7 +1684,7 @@ dsv4-fp4-mi355x-sglang-agentic-hicache: - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 + image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1695,8 +1695,10 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64], spec-decoding: mtp } - - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], spec-decoding: mtp } + #- { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64], spec-decoding: mtp } + # remove + #- { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [72], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From e5528b7be90b220f4166b612479f1375dcc4a13d Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 18:08:48 +0900 Subject: [PATCH 33/35] [AMD][AgentX] dsv4-fp4-mi355x-sglang-agentic-mtp: revert sglang image to v0.5.15.post1 and set kvnone conc-list to [64] Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 488b9c5fea..1dbc8beb41 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1684,7 +1684,7 @@ dsv4-fp4-mi355x-sglang-agentic-hicache: - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260729 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1695,10 +1695,8 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - #- { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64], spec-decoding: mtp } - # remove - #- { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], spec-decoding: mtp } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [72], spec-decoding: mtp } + #- { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 8, 16, 32, 48, 64], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [64], spec-decoding: mtp } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 From 180e5084eb140c7942233be1b8551300f6d88503 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 31 Jul 2026 13:27:29 +0900 Subject: [PATCH 34/35] [AMD][AgentX] pin aiperf venv to Python 3.11 so aiperf>=0.12 (requires-python >=3.11) installs on the mi355x sglang-rocm image (default python3 is 3.10) Co-Authored-By: Claude Opus 4.6 --- benchmarks/benchmark_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 92d0c2eac4..dfb3f7aafc 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1624,7 +1624,7 @@ install_agentic_deps() { mkdir -p "$AIPERF_UV_CACHE_DIR" UV_CACHE_DIR="$AIPERF_UV_CACHE_DIR" \ - "$AIPERF_UV_BIN" venv --python "$(command -v python3)" "$AIPERF_VENV" + "$AIPERF_UV_BIN" venv --python "${AIPERF_PYTHON_VERSION:-3.11}" "$AIPERF_VENV" UV_CACHE_DIR="$AIPERF_UV_CACHE_DIR" \ "$AIPERF_UV_BIN" pip install --python "$AIPERF_PYTHON" \ -r "$AGENTIC_DIR/requirements.txt" \ From dbc1153b622fc7ee0bdbac4f45d7afe1b73dc431 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 31 Jul 2026 16:43:56 +0900 Subject: [PATCH 35/35] [AMD][AgentX] dsv4 mi355x sglang mtp: use cache_aware router policy for prefix-affinity across DP ranks consistent_hashing keyed on per-request-unique x-correlation-id scattered a session's turns across DP ranks, so the radix cache could not be reused (GPU cache hit ~67% vs ~97% theoretical at conc=64). Switch to cache_aware (radix-tree longest-prefix match per DP rank) and drop the forced correlation-id routing key so routing follows request content. Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 9b6557fa53..c0c1d88de1 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -87,7 +87,6 @@ CHUNKED_PREFILL_SIZE=4096 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true SGLANG_BACKEND_PORT=$((PORT + 1)) SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) @@ -194,9 +193,20 @@ wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" - if [ "$USE_SGLANG_ROUTER" = "true" ]; then echo "Starting SGLang router on port $PORT for $TP DP ranks..." + # cache_aware routes each request to the DP rank whose radix tree holds the + # longest matching prefix, giving prefix-affinity across DP ranks. This + # replaces consistent_hashing on x-correlation-id: that key is per-request + # unique, so turns of the same session hashed to different ranks and the + # radix cache could not be reused (GPU cache hit ~67% vs ~97% theoretical). + # --cache-threshold: min prefix-match ratio to prefer the cached rank; + # below it, fall back to load balancing. --balance-*-threshold bound how far + # cache affinity may skew per-rank load before rebalancing. "${SGLANG_ROUTER_CMD[@]}" \ --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ + --policy cache_aware \ + --cache-threshold 0.3 \ + --balance-abs-threshold 64 \ + --balance-rel-threshold 1.5 \ --request-id-headers x-correlation-id \ --dp-aware \ --host 0.0.0.0 \