From 52d7a5603ae1f68e1bce58a77a0c626217cdc86b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:48:20 -0500
Subject: [PATCH 01/32] feat: add Qwen3.5 FP4 B200 AgentX MTP
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the initial SGLang NEXTN AgentX recipe with golden synthetic acceptance and 256k traces.\n\n中文：新增 Qwen3.5 FP4 B200 AgentX MTP 初始配置，使用 SGLang NEXTN、黄金合成接受长度和 256k 轨迹数据集。
---
 .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh    | 118 ++++++++++++++++++
 configs/nvidia-master.yaml                    |  15 +++
 2 files changed, 133 insertions(+)
 create mode 100755 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
new file mode 100755
index 0000000000..e29492a0c4
--- /dev/null
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# AgentX trace replay for Qwen3.5-397B-A17B NVFP4 on B200 with SGLang
+# native NEXTN MTP. Throughput uses the committed golden synthetic AL; evals
+# retain real target-model verification.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL TP CONC EP_SIZE KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
+
+SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+nvidia-smi
+
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k
+resolve_trace_source
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
+    MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
+    HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
+    if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
+        echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB" >&2
+        exit 1
+    fi
+    CACHE_ARGS=(
+        --page-size 64
+        --enable-hierarchical-cache
+        --hicache-size "$HICACHE_SIZE_GB"
+        --hicache-io-backend kernel
+        --hicache-mem-layout page_first
+        --hicache-write-policy write_through_selective
+    )
+fi
+
+export TORCH_CUDA_ARCH_LIST="10.0"
+export PYTHONNOUSERSITE=1
+export NCCL_NVLS_ENABLE=1
+export SGL_ENABLE_JIT_DEEPGEMM=false
+export SGLANG_ENABLE_FLASHINFER_GEMM=true
+export SGLANG_ENABLE_SPEC_V2=1
+
+if [ "${EVAL_ONLY:-false}" != "true" ]; then
+    export SGLANG_SIMULATE_ACC_LEN=3.39
+    export SGLANG_SIMULATE_ACC_METHOD=match-expected
+    export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
+fi
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    --tensor-parallel-size "$TP"
+    --data-parallel-size 1
+    --expert-parallel-size "$EP_SIZE"
+    --enable-symm-mem
+    --quantization modelopt_fp4
+    --fp4-gemm-backend flashinfer_cutlass
+    --kv-cache-dtype fp8_e4m3
+    --mamba-ssm-dtype bfloat16
+    --attention-backend trtllm_mha
+    --moe-runner-backend flashinfer_trtllm
+    --cuda-graph-max-bs "$CONC"
+    --max-running-requests "$CONC"
+    --max-prefill-tokens 16384
+    --chunked-prefill-size 16384
+    --mem-fraction-static 0.80
+    --stream-interval 50
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
+    --tokenizer-worker-num 6
+    --tokenizer-path "$MODEL"
+    --reasoning-parser qwen3
+    --tool-call-parser qwen3_coder
+    --enable-auto-tool-choice
+    --speculative-algorithm NEXTN
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+    --enable-metrics
+    "${CACHE_ARGS[@]}"
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "${EVAL_ONLY:-false}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 8ee037c8c2..0a82dcf938 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1249,6 +1249,21 @@ qwen3.5-fp8-b200-sglang-agentic:
     - search-space:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
 
+qwen3.5-fp4-b200-sglang-agentic-mtp:
+  image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: cluster:b200-dgxc
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16] }
+
 qwen3.5-fp4-b200-sglang:
   image: lmsysorg/sglang:v0.5.14-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4

From bf03912d13ffd7264005398ef4be52aacbab315d Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:48:49 -0500
Subject: [PATCH 02/32] chore: trigger B200 AgentX sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Append the Qwen3.5 FP4 B200 AgentX MTP benchmark entry.\n\n中文：追加 Qwen3.5 FP4 B200 AgentX MTP 基准测试触发记录。
---
 perf-changelog.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index c1736ae420..8147101761 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5311,3 +5311,10 @@
     - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs."
     - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418
+
+- config-keys:
+    - qwen3.5-fp4-b200-sglang-agentic-mtp
+  description:
+    - "Add Qwen3.5-397B-A17B NVFP4 AgentX benchmark on B200 with SGLang native NEXTN MTP"
+    - "Use the 256k trace dataset and golden synthetic acceptance length 3.39"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420

From 9c682e2959df320a071f847291923d685680125d Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:50:42 -0500
Subject: [PATCH 03/32] perf: expand B200 AgentX preflight range

---
 configs/nvidia-master.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 0a82dcf938..3adacdae7f 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1261,8 +1261,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
 
 qwen3.5-fp4-b200-sglang:
   image: lmsysorg/sglang:v0.5.14-cu130

From 2d6c12b17594f7ac7c51bd0ea8fdf9046f193bc6 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:54:43 -0500
Subject: [PATCH 04/32] chore: use SGLang v0.5.16 for Qwen B200

---
 configs/nvidia-master.yaml | 2 +-
 perf-changelog.yaml        | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 3adacdae7f..95c89dcb23 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1250,7 +1250,7 @@ qwen3.5-fp8-b200-sglang-agentic:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
 
 qwen3.5-fp4-b200-sglang-agentic-mtp:
-  image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1
+  image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
   model-prefix: qwen3.5
   runner: cluster:b200-dgxc
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8147101761..d681cc61bb 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5317,4 +5317,5 @@
   description:
     - "Add Qwen3.5-397B-A17B NVFP4 AgentX benchmark on B200 with SGLang native NEXTN MTP"
     - "Use the 256k trace dataset and golden synthetic acceptance length 3.39"
+    - "Image: lmsysorg/sglang:v0.5.16-cu130"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420

From b118256207ec6919cc891711d941bb8458e51670 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:55:38 -0500
Subject: [PATCH 05/32] chore: collect SGLang cache metrics on B200

---
 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index e29492a0c4..dc5a6e2322 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -114,5 +114,6 @@ if [ "${EVAL_ONLY:-false}" = "true" ]; then
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi

From e2c4ab50d683dcccbe2f59ad387e422631f44383 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 29 Jul 2026 23:59:58 -0500
Subject: [PATCH 06/32] fix: remove unsupported SGLang tool-choice flag on B200

---
 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index dc5a6e2322..d93700c82b 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -94,7 +94,6 @@ SGLANG_CMD=(
     --tokenizer-path "$MODEL"
     --reasoning-parser qwen3
     --tool-call-parser qwen3_coder
-    --enable-auto-tool-choice
     --speculative-algorithm NEXTN
     --speculative-num-steps 3
     --speculative-eagle-topk 1

From 4a587bd5a3876748da4ed1e038b45ec71718646f Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 00:42:56 -0500
Subject: [PATCH 07/32] perf: prefer local Qwen FP4 weights on B200
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Probe the allocated DGXC node for the complete Qwen3.5 NVFP4 checkpoint on /raid and fall back to shared Lustre when it is absent.\n\n中文：在分配到的 DGXC 节点上探测 /raid 中完整的 Qwen3.5 NVFP4 权重；若未暂存，则回退到共享 Lustre。
---
 runners/launch_b200-dgxc.sh | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh
index b304ad4c1f..95924652d6 100644
--- a/runners/launch_b200-dgxc.sh
+++ b/runners/launch_b200-dgxc.sh
@@ -11,8 +11,8 @@ set -x
 # portability, but we resolve to /lustre/fsw/models/* here to avoid repeated
 # downloading on every dgxc node. Runs for both single-node and multinode
 # launches.
-# NOTE: per-node /raid/models/* would be faster but is only populated on a
-# subset of dgxc nodes today, so we use Lustre for reliability.
+# Single-node Qwen3.5 FP4 runs probe the allocated node for a complete local
+# /raid checkpoint below and use this shared path only as the fallback.
 if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2"
     export SRT_SLURM_MODEL_PREFIX="dsr1"
@@ -437,10 +437,6 @@ EOF
 else
 
     SQUASH_FILE="/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
-    # Point the bench script at the local MODEL_PATH resolved above instead of
-    # pulling from the HF hub cache. Bench scripts skip `hf download` when
-    # MODEL is a local path.
-    export MODEL="$MODEL_PATH"
     FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '')
     SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '')
     # Prefer a framework-tagged script (e.g. dsv4_fp4_b200_vllm.sh) so models
@@ -474,6 +470,28 @@ else
     salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
 
+    # Qwen3.5 FP4 is staged on local NVMe on some, but not all, DGXC nodes.
+    # Resolve this only after Slurm assigns the node; checking on the login
+    # host would select the wrong filesystem. A config plus the safetensor
+    # index are required so a partial staging directory is never selected.
+    if [[ "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" ]]; then
+        LOCAL_MODEL_PATH="/raid/models/Qwen3.5-397B-A17B-NVFP4"
+        if srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 \
+            test -f "$LOCAL_MODEL_PATH/config.json" -a \
+                 -f "$LOCAL_MODEL_PATH/model.safetensors.index.json"; then
+            MODEL_PATH="$LOCAL_MODEL_PATH"
+            export MODEL_PATH
+            echo "Using node-local NVMe checkpoint: $MODEL_PATH"
+        else
+            echo "Node-local Qwen3.5 FP4 checkpoint unavailable; using shared checkpoint: $MODEL_PATH"
+        fi
+    fi
+
+    # Point the bench script at the resolved local/shared MODEL_PATH instead of
+    # pulling from the HF hub cache. Bench scripts skip `hf download` when
+    # MODEL is a local path.
+    export MODEL="$MODEL_PATH"
+
     # Use flock to serialize concurrent imports to the same squash file
     # Override ENROOT_CACHE_PATH to avoid permission issues with system-wide cache on worker nodes
     srun --jobid=$JOB_ID bash -c "

From 1b863c55f1449aca2a0957602cd57970654d3e3a Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 00:48:48 -0500
Subject: [PATCH 08/32] perf: prepare Qwen AgentX cache and DEP tuning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add strict two-pool HiCache accounting, request cache reporting, session-sticky attention-DP routing, and AgentX request/graph limits for targeted B200 tuning.\n\n中文：为 B200 定向调优加入严格的双池 HiCache 容量核算、请求级缓存统计、会话粘性的注意力数据并行路由，以及 AgentX 请求数与 CUDA Graph 上限。
---
 .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh    | 101 +++++++++++++++---
 1 file changed, 89 insertions(+), 12 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index d93700c82b..8922283d56 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -9,7 +9,8 @@ set -x
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars \
-    MODEL TP CONC EP_SIZE KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
+    MODEL TP CONC EP_SIZE DP_ATTENTION KV_OFFLOADING \
+    TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
 
 SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
 
@@ -36,6 +37,12 @@ mkdir -p "$RESULT_DIR"
 
 CACHE_ARGS=()
 if require_agentic_kv_offload_backend hicache; then
+    REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
+    if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
+        echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
+        exit 1
+    fi
+    TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
     HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
     MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
     HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
@@ -43,6 +50,7 @@ if require_agentic_kv_offload_backend hicache; then
         echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB" >&2
         exit 1
     fi
+    echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank per pool across TP=${TP}, pools=${HICACHE_HOST_POOL_COUNT}; node total <= ${TOTAL_CPU_DRAM_GB} GB"
     CACHE_ARGS=(
         --page-size 64
         --enable-hierarchical-cache
@@ -53,12 +61,51 @@ if require_agentic_kv_offload_backend hicache; then
     )
 fi
 
+USE_SGLANG_ROUTER=false
+SGLANG_BACKEND_PORT="$PORT"
+ROUTER_LOG="$RESULT_DIR/router.log"
+if [ "$DP_ATTENTION" = "true" ]; then
+    USE_SGLANG_ROUTER=true
+    export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
+    SGLANG_BACKEND_PORT=$((PORT + 1))
+    SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
+    SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router)
+fi
+
+PARALLEL_ARGS=(
+    --tp "$TP"
+    --dp 1
+    --ep-size "$EP_SIZE"
+)
+TOKENIZER_WORKERS=6
+STREAM_INTERVAL=50
+if [ "$DP_ATTENTION" = "true" ]; then
+    PARALLEL_ARGS=(
+        --tp "$TP"
+        --dp "$TP"
+        --ep-size "$EP_SIZE"
+        --enable-dp-attention
+        --enable-dp-attention-local-control-broadcast
+        --enable-dp-lm-head
+        --dist-init-addr "127.0.0.1:$((PORT + 2000))"
+        --incremental-streaming-output
+    )
+    TOKENIZER_WORKERS="$TP"
+    STREAM_INTERVAL=20
+fi
+
+# AgentX concurrency counts live session trees rather than individual HTTP
+# requests. Leave room for subagent fan-out and avoid spending HBM on graphs
+# above the batch sizes that remain useful for this long-context workload.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export NCCL_NVLS_ENABLE=1
 export SGL_ENABLE_JIT_DEEPGEMM=false
 export SGLANG_ENABLE_FLASHINFER_GEMM=true
-export SGLANG_ENABLE_SPEC_V2=1
 
 if [ "${EVAL_ONLY:-false}" != "true" ]; then
     export SGLANG_SIMULATE_ACC_LEN=3.39
@@ -71,11 +118,9 @@ SGLANG_CMD=(
     --model-path "$MODEL_PATH"
     --served-model-name "$MODEL"
     --host 0.0.0.0
-    --port "$PORT"
+    --port "$SGLANG_BACKEND_PORT"
     --trust-remote-code
-    --tensor-parallel-size "$TP"
-    --data-parallel-size 1
-    --expert-parallel-size "$EP_SIZE"
+    "${PARALLEL_ARGS[@]}"
     --enable-symm-mem
     --quantization modelopt_fp4
     --fp4-gemm-backend flashinfer_cutlass
@@ -83,14 +128,14 @@ SGLANG_CMD=(
     --mamba-ssm-dtype bfloat16
     --attention-backend trtllm_mha
     --moe-runner-backend flashinfer_trtllm
-    --cuda-graph-max-bs "$CONC"
-    --max-running-requests "$CONC"
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --max-prefill-tokens 16384
     --chunked-prefill-size 16384
     --mem-fraction-static 0.80
-    --stream-interval 50
+    --stream-interval "$STREAM_INTERVAL"
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
-    --tokenizer-worker-num 6
+    --tokenizer-worker-num "$TOKENIZER_WORKERS"
     --tokenizer-path "$MODEL"
     --reasoning-parser qwen3
     --tool-call-parser qwen3_coder
@@ -99,6 +144,7 @@ SGLANG_CMD=(
     --speculative-eagle-topk 1
     --speculative-num-draft-tokens 4
     --enable-metrics
+    --enable-cache-report
     "${CACHE_ARGS[@]}"
 )
 
@@ -107,12 +153,43 @@ printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
 "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+capture_cache_metrics() {
+    {
+        echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
+        curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \
+            | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \
+            || true
+        echo "============================================================"
+    } >> "$SERVER_LOG"
+}
+
+wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "$USE_SGLANG_ROUTER" = "true" ]; then
+    "${SGLANG_ROUTER_CMD[@]}" \
+        --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
+        --policy consistent_hashing \
+        --request-id-headers x-correlation-id \
+        --dp-aware \
+        --host 0.0.0.0 \
+        --port "$PORT" \
+        --prometheus-host 127.0.0.1 \
+        --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
+        --connect-timeout-secs 900 \
+        --request-timeout-secs 14400 \
+        --disable-health-check \
+        --disable-retries > "$ROUTER_LOG" 2>&1 &
+    ROUTER_PID=$!
+    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
+fi
+
+capture_cache_metrics
+trap capture_cache_metrics EXIT
 
 if [ "${EVAL_ONLY:-false}" = "true" ]; then
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
-    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
+    REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi

From 1e897897514e0bb9e9ba9377a83b32967a396850 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 00:58:59 -0500
Subject: [PATCH 09/32] perf: add Qwen B200 DEP probes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move the configuration into the AgentX section, document the measured cache model, and add session-sticky DEP8 no-offload probes for the throughput frontier.\n\n中文：将配置移入 AgentX 区域，记录实测缓存模型，并为吞吐量前沿加入具备会话粘性路由的 DEP8 无卸载探测点。
---
 configs/nvidia-master.yaml | 39 +++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 95c89dcb23..557aa1ec35 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1249,21 +1249,6 @@ qwen3.5-fp8-b200-sglang-agentic:
     - search-space:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
 
-qwen3.5-fp4-b200-sglang-agentic-mtp:
-  image: lmsysorg/sglang:v0.5.16-cu130
-  model: nvidia/Qwen3.5-397B-A17B-NVFP4
-  model-prefix: qwen3.5
-  runner: cluster:b200-dgxc
-  precision: fp4
-  framework: sglang
-  multinode: false
-  scenarios:
-    agentic-coding:
-    - dram-utilization: 0.80
-      search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
-
 qwen3.5-fp4-b200-sglang:
   image: lmsysorg/sglang:v0.5.14-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8457,6 +8442,30 @@ minimaxm3-fp8-h200-vllm-agentic:
       - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
       - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
 
+# Qwen3.5 has 15 full-attention layers with two 256-wide KV heads, so its
+# target FP8 KV is 15,360 bytes/token before the native MTP layer. TP4
+# replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
+# measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
+# p75 ISL, the measured 2.24M-token TP2 pool should bend around conc 12-24,
+# while the 8.37M-token TP4 pool should bend around conc 48-72. DEP8 covers
+# the throughput end by sharding experts while routing each session tree
+# consistently to one attention-DP rank.
+qwen3.5-fp4-b200-sglang-agentic-mtp:
+  image: lmsysorg/sglang:v0.5.16-cu130
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  model-prefix: qwen3.5
+  runner: cluster:b200-dgxc
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
+      - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" } }
+
 dsv4-fp4-b200-sglang-agentic-hicache:
   image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233
   model: deepseek-ai/DeepSeek-V4-Pro

From f7e3b3768804fccc263e1907e484b562e2376b7b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:13:06 -0500
Subject: [PATCH 10/32] perf: add matched B200 HiCache cliff probe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：添加 B200 TP2 c16 HiCache 对照探针，并严格使用按 GPU 数量计算的 DRAM 上限。
---
 configs/nvidia-master.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 557aa1ec35..7ad16f0737 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8464,6 +8464,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
+      # Matched cliff probe: TP2 c16 reaches ~70% GPU KV usage without offload.
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16] }
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" } }
 
 dsv4-fp4-b200-sglang-agentic-hicache:

From 191ebbea45cdd923bb285b10fbd6a84d58b79e18 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:13:41 -0500
Subject: [PATCH 11/32] perf: isolate B200 HiCache validation point
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：将 B200 TP2 HiCache 验证点设为独立的 c20，避免一次性重复启动无卸载配置。
---
 configs/nvidia-master.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 7ad16f0737..7f6096979f 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8464,8 +8464,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
-      # Matched cliff probe: TP2 c16 reaches ~70% GPU KV usage without offload.
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16] }
+      # Near-cliff HiCache probe: TP2 c16 reaches ~70% GPU KV usage without offload.
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20] }
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" } }
 
 dsv4-fp4-b200-sglang-agentic-hicache:

From b439bf7dc192f21fa098fac22419c59e9d613850 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:38:30 -0500
Subject: [PATCH 12/32] perf: extend Qwen B200 TP4 cliff probe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the isolated TP4 c96 no-offload point after c64 remained healthy, so the HBM working-set cliff can be measured before adding matched HiCache coverage.

中文：在 c64 保持健康后新增独立的 TP4 c96 无卸载测试点，以便在添加匹配的 HiCache 覆盖前测量 HBM 工作集拐点。
---
 configs/nvidia-master.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 7f6096979f..42244f629e 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8462,7 +8462,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64, 96] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
       # Near-cliff HiCache probe: TP2 c16 reaches ~70% GPU KV usage without offload.
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20] }

From df5a38810996b940a358850b89a6a7b2b1eb47ed Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:42:05 -0500
Subject: [PATCH 13/32] perf: drop dominated Qwen B200 DEP arm
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove the DEP8 search arm after its c128 probe fragmented prefix reuse, completed no profiled requests in three minutes, and delivered less than half of TP4 c64 aggregate prefill throughput.

中文：DEP8 c128 测试出现前缀复用碎片化，三分钟内没有完成任何正式请求，聚合预填充吞吐量不足 TP4 c64 的一半，因此移除该搜索分支。
---
 configs/nvidia-master.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 42244f629e..8e19a2b68e 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8447,9 +8447,7 @@ minimaxm3-fp8-h200-vllm-agentic:
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
 # p75 ISL, the measured 2.24M-token TP2 pool should bend around conc 12-24,
-# while the 8.37M-token TP4 pool should bend around conc 48-72. DEP8 covers
-# the throughput end by sharding experts while routing each session tree
-# consistently to one attention-DP rank.
+# while the 8.37M-token TP4 pool should bend around conc 48-72.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8466,7 +8464,6 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
       # Near-cliff HiCache probe: TP2 c16 reaches ~70% GPU KV usage without offload.
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20] }
-      - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" } }
 
 dsv4-fp4-b200-sglang-agentic-hicache:
   image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233

From ce0528605b537b686d1c88dadb52391dcae89541 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:45:52 -0500
Subject: [PATCH 14/32] fix: enforce Qwen HiCache DRAM total
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Account for SGLang target KV, Mamba, and native NEXTN draft host pools as H * 31/15 per rank, with page-alignment reserve, so projected use stays within runner-generated DRAM.

中文：按每个 rank 的 H * 31/15 计入 SGLang 目标 KV、Mamba 与原生 NEXTN 草稿主机缓存，并预留页对齐空间，确保预计用量不超过 runner 生成的 DRAM 上限。
---
 .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh    | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index 8922283d56..47cd7d8c4a 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -43,14 +43,28 @@ if require_agentic_kv_offload_backend hicache; then
         exit 1
     fi
     TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
-    HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
-    MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
+    # SGLang applies --hicache-size independently to Qwen's target KV and
+    # Mamba pools. Native NEXTN also creates a draft KV pool with the same
+    # slot count; its one attention layer adds 1/15 of the target KV bytes.
+    # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank.
+    HICACHE_ALIGNMENT_RESERVE_GB=$TP
+    HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB))
+    if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then
+        echo "Error: insufficient DRAM after HiCache alignment reserve" >&2
+        exit 1
+    fi
+    MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31))
     HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
     if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
         echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB" >&2
         exit 1
     fi
-    echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB per rank per pool across TP=${TP}, pools=${HICACHE_HOST_POOL_COUNT}; node total <= ${TOTAL_CPU_DRAM_GB} GB"
+    PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB))
+    if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
+        echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
+        exit 1
+    fi
+    echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB"
     CACHE_ARGS=(
         --page-size 64
         --enable-hierarchical-cache

From 718ff2bc4edfd20da66f75800bdba9d4d6b48e1e Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:47:15 -0500
Subject: [PATCH 15/32] perf: add B200 TP4 c72 HiCache A/B
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add matched no-offload and DRAM HiCache c72 points immediately after the measured c64 KV-pressure onset.

中文：在实测 c64 KV 压力拐点之后，添加匹配的 c72 无卸载与 DRAM HiCache 对照测试点。
---
 configs/nvidia-master.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 8e19a2b68e..7f1e1133d3 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8460,7 +8460,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64, 96] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64, 72, 96] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [72] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
       # Near-cliff HiCache probe: TP2 c16 reaches ~70% GPU KV usage without offload.
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20] }

From efbe79d5d7844ef6c9bf764e6a8b9caa51a31185 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 01:58:03 -0500
Subject: [PATCH 16/32] refactor: remove dominated B200 DEP path
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Simplify the B200 launcher script to TP-only serving after the measured DEP arm was removed from the search space.

中文：实测 DEP 分支已从搜索空间移除，因此将 B200 启动脚本简化为仅使用 TP 的服务路径。
---
 .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh    | 59 +++----------------
 1 file changed, 7 insertions(+), 52 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index 47cd7d8c4a..35474f4917 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -9,7 +9,7 @@ set -x
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars \
-    MODEL TP CONC EP_SIZE DP_ATTENTION KV_OFFLOADING \
+    MODEL TP CONC EP_SIZE KV_OFFLOADING \
     TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
 
 SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
@@ -75,38 +75,11 @@ if require_agentic_kv_offload_backend hicache; then
     )
 fi
 
-USE_SGLANG_ROUTER=false
-SGLANG_BACKEND_PORT="$PORT"
-ROUTER_LOG="$RESULT_DIR/router.log"
-if [ "$DP_ATTENTION" = "true" ]; then
-    USE_SGLANG_ROUTER=true
-    export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
-    SGLANG_BACKEND_PORT=$((PORT + 1))
-    SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
-    SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router)
-fi
-
 PARALLEL_ARGS=(
     --tp "$TP"
     --dp 1
     --ep-size "$EP_SIZE"
 )
-TOKENIZER_WORKERS=6
-STREAM_INTERVAL=50
-if [ "$DP_ATTENTION" = "true" ]; then
-    PARALLEL_ARGS=(
-        --tp "$TP"
-        --dp "$TP"
-        --ep-size "$EP_SIZE"
-        --enable-dp-attention
-        --enable-dp-attention-local-control-broadcast
-        --enable-dp-lm-head
-        --dist-init-addr "127.0.0.1:$((PORT + 2000))"
-        --incremental-streaming-output
-    )
-    TOKENIZER_WORKERS="$TP"
-    STREAM_INTERVAL=20
-fi
 
 # AgentX concurrency counts live session trees rather than individual HTTP
 # requests. Leave room for subagent fan-out and avoid spending HBM on graphs
@@ -132,7 +105,7 @@ SGLANG_CMD=(
     --model-path "$MODEL_PATH"
     --served-model-name "$MODEL"
     --host 0.0.0.0
-    --port "$SGLANG_BACKEND_PORT"
+    --port "$PORT"
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
     --enable-symm-mem
@@ -147,9 +120,9 @@ SGLANG_CMD=(
     --max-prefill-tokens 16384
     --chunked-prefill-size 16384
     --mem-fraction-static 0.80
-    --stream-interval "$STREAM_INTERVAL"
+    --stream-interval 50
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
-    --tokenizer-worker-num "$TOKENIZER_WORKERS"
+    --tokenizer-worker-num 6
     --tokenizer-path "$MODEL"
     --reasoning-parser qwen3
     --tool-call-parser qwen3_coder
@@ -170,32 +143,14 @@ SERVER_PID=$!
 capture_cache_metrics() {
     {
         echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
-        curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \
+        curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \
             | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \
             || true
         echo "============================================================"
     } >> "$SERVER_LOG"
 }
 
-wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "$USE_SGLANG_ROUTER" = "true" ]; then
-    "${SGLANG_ROUTER_CMD[@]}" \
-        --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
-        --policy consistent_hashing \
-        --request-id-headers x-correlation-id \
-        --dp-aware \
-        --host 0.0.0.0 \
-        --port "$PORT" \
-        --prometheus-host 127.0.0.1 \
-        --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
-        --connect-timeout-secs 900 \
-        --request-timeout-secs 14400 \
-        --disable-health-check \
-        --disable-retries > "$ROUTER_LOG" 2>&1 &
-    ROUTER_PID=$!
-    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
 capture_cache_metrics
 trap capture_cache_metrics EXIT
@@ -204,6 +159,6 @@ if [ "${EVAL_ONLY:-false}" = "true" ]; then
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
-    REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi

From 0eddccecc1349278f3d953f8f99180b04b5b8617 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:05:57 -0500
Subject: [PATCH 17/32] perf: densify B200 TP2 cache cliff
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the dominated c24-c64 no-offload tail with dense c12-c22 points and matched HiCache coverage beginning at c16.

中文：移除性能受压的 c24-c64 无卸载尾部，改为密集采样 c12-c22，并从 c16 开始提供匹配的 HiCache 覆盖。
---
 configs/nvidia-master.yaml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index d2e581e5d1..d7b41f1238 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8482,8 +8482,8 @@ minimaxm3-fp8-h200-vllm-agentic:
 # target FP8 KV is 15,360 bytes/token before the native MTP layer. TP4
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
-# p75 ISL, the measured 2.24M-token TP2 pool should bend around conc 12-24,
-# while the 8.37M-token TP4 pool should bend around conc 48-72.
+# p75 ISL, measured TP2 reaches 98% GPU KV at c16 and collapses by c24;
+# TP4 reaches 95% GPU KV at c64 while retaining useful throughput.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8498,9 +8498,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64, 72, 96] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [72] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 48, 64] }
-      # Near-cliff HiCache probe: TP2 c16 reaches ~70% GPU KV usage without offload.
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20, 22] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }
 
 dsv4-fp4-b200-sglang-agentic-hicache:
   image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233

From 123f0f882bb6af1abd132556a7bd503ab3a8d1d1 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:13:12 -0500
Subject: [PATCH 18/32] perf: resolve B200 Qwen cache cliff
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Densify TP4 around the measured c64-c72 transition and extend the HiCache curve beyond the HBM cliff.\n\n中文：细化 B200 Qwen TP4 在实测 c64-c72 缓存拐点附近的并发点，并将 HiCache 曲线扩展到 HBM 拐点之后。
---
 configs/nvidia-master.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index d7b41f1238..8fd84cb546 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8482,8 +8482,9 @@ minimaxm3-fp8-h200-vllm-agentic:
 # target FP8 KV is 15,360 bytes/token before the native MTP layer. TP4
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
-# p75 ISL, measured TP2 reaches 98% GPU KV at c16 and collapses by c24;
-# TP4 reaches 95% GPU KV at c64 while retaining useful throughput.
+# p75 ISL, measured TP2 reaches 98% GPU KV at c16 and collapses by c24.
+# TP4 retains 83.9% prefix hits at c64 but falls to 0% at c72, so the
+# no-offload grid resolves c64-70 densely and HiCache takes over at the cliff.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8496,8 +8497,8 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 64, 72, 96] }
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [72] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68, 70] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20, 22] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }
 

From b2477e47db15a6d7ad5f161977c01e68b9a510d5 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:24:37 -0500
Subject: [PATCH 19/32] perf: prune dominated B200 TP2 point
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove no-offload c22 after live profiling placed it past the TP2 cache cliff, while retaining the HiCache arm and c20 boundary.\n\n中文：实测表明无卸载 c22 已越过 TP2 缓存拐点，因此移除该点，同时保留 HiCache 配置与 c20 边界点。
---
 configs/nvidia-master.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 8fd84cb546..11e0d0e97b 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8482,7 +8482,7 @@ minimaxm3-fp8-h200-vllm-agentic:
 # target FP8 KV is 15,360 bytes/token before the native MTP layer. TP4
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
-# p75 ISL, measured TP2 reaches 98% GPU KV at c16 and collapses by c24.
+# p75 ISL, TP2 remains useful at c18 but is dominated by c22.
 # TP4 retains 83.9% prefix hits at c64 but falls to 0% at c72, so the
 # no-offload grid resolves c64-70 densely and HiCache takes over at the cliff.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
@@ -8499,7 +8499,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68, 70] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20, 22] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }
 
 dsv4-fp4-b200-sglang-agentic-hicache:

From 3ecfbc8b4f6fd0dcd8f213d16ccd781763c8c49b Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:39:41 -0500
Subject: [PATCH 20/32] perf: stop B200 TP4 before cache thrashing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：在缓存抖动前收紧 B200 TP4 并发范围。
---
 configs/nvidia-master.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 11e0d0e97b..f8c3be3668 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8483,8 +8483,9 @@ minimaxm3-fp8-h200-vllm-agentic:
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
 # p75 ISL, TP2 remains useful at c18 but is dominated by c22.
-# TP4 retains 83.9% prefix hits at c64 but falls to 0% at c72, so the
-# no-offload grid resolves c64-70 densely and HiCache takes over at the cliff.
+# TP4 retains 83.9% prefix hits at c64, remains useful at c66, but c70 falls
+# to 6% hits and 96% KV usage. Resolve c64-68 densely and let HiCache take
+# over across the cliff.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8497,7 +8498,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68, 70] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }

From e518bc00ff0731658893c788e71c66ceebb2ec9c Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:42:12 -0500
Subject: [PATCH 21/32] perf: switch B200 TP2 to HiCache after c14
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：B200 TP2 在并发 14 后切换至 HiCache。
---
 configs/nvidia-master.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index f8c3be3668..e4d1fa9637 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8482,7 +8482,9 @@ minimaxm3-fp8-h200-vllm-agentic:
 # target FP8 KV is 15,360 bytes/token before the native MTP layer. TP4
 # replicates the two KV heads and measured 30.5 KB/token node-wide; TP2
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
-# p75 ISL, TP2 remains useful at c18 but is dominated by c22.
+# p75 ISL, TP2 no-offload is already GPU-KV-bound by c18; measured c20
+# HiCache improves both its throughput and latency, so offload takes over
+# after c14 rather than repeating dominated no-offload points.
 # TP4 retains 83.9% prefix hits at c64, remains useful at c66, but c70 falls
 # to 6% hits and 96% KV usage. Resolve c64-68 densely and let HiCache take
 # over across the cliff.
@@ -8500,7 +8502,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14, 16, 18, 20] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }
 
 dsv4-fp4-b200-sglang-agentic-hicache:

From a7b78e1f4e2a8560051d78678aad846786201f1a Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:43:30 -0500
Subject: [PATCH 22/32] perf: prune dominated B200 TP4 c66
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：裁剪性能受支配的 B200 TP4 并发 66 配置。
---
 configs/nvidia-master.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index e4d1fa9637..2fdec091d8 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8485,9 +8485,9 @@ minimaxm3-fp8-h200-vllm-agentic:
 # p75 ISL, TP2 no-offload is already GPU-KV-bound by c18; measured c20
 # HiCache improves both its throughput and latency, so offload takes over
 # after c14 rather than repeating dominated no-offload points.
-# TP4 retains 83.9% prefix hits at c64, remains useful at c66, but c70 falls
-# to 6% hits and 96% KV usage. Resolve c64-68 densely and let HiCache take
-# over across the cliff.
+# TP4 retains 83.9% prefix hits at c64; c66 is already dominated, while c70
+# falls to 6% hits and 96% KV usage. Keep c68 as the final no-offload cliff
+# probe and let HiCache take over across the boundary.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8500,7 +8500,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 66, 68] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 68] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }

From 28ac3e6d871890263fada4b95756e173e374fdf1 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 02:52:53 -0500
Subject: [PATCH 23/32] fix(agentx): use stable tokenizer startup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：使用 SGLang 默认单 tokenizer 启动路径，避免长时间 HiCache 初始化期间的多 tokenizer 共享内存竞态。
---
 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index 35474f4917..40aa2d76d1 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -122,7 +122,6 @@ SGLANG_CMD=(
     --mem-fraction-static 0.80
     --stream-interval 50
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
-    --tokenizer-worker-num 6
     --tokenizer-path "$MODEL"
     --reasoning-parser qwen3
     --tool-call-parser qwen3_coder

From 801f92216fa4bc9e296f4b35db9a042cda808708 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 03:01:31 -0500
Subject: [PATCH 24/32] perf(agentx): cap B200 no-offload frontier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：移除已被 c64 严格支配的 B200 c68 无卸载点，并让 HiCache 从缓存悬崖边界开始接管。
---
 configs/nvidia-master.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 2fdec091d8..af013b6902 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8485,9 +8485,9 @@ minimaxm3-fp8-h200-vllm-agentic:
 # p75 ISL, TP2 no-offload is already GPU-KV-bound by c18; measured c20
 # HiCache improves both its throughput and latency, so offload takes over
 # after c14 rather than repeating dominated no-offload points.
-# TP4 retains 83.9% prefix hits at c64; c66 is already dominated, while c70
-# falls to 6% hits and 96% KV usage. Keep c68 as the final no-offload cliff
-# probe and let HiCache take over across the boundary.
+# TP4 retains 83.9% prefix hits at c64. No-offload c66 and c68 are already
+# throughput/latency dominated, while c70 collapses to 6% hits and 96% KV
+# usage, so HiCache takes over at c64 and densely covers the boundary.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8500,7 +8500,7 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64, 68] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] }
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }

From 8d05af85df959c71c99f99f5658919d073019191 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 03:19:34 -0500
Subject: [PATCH 25/32] fix(agentx): scale Qwen tokenization by topology
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Use six tokenizer workers for TP4 long-context replay while retaining SGLang default single-worker startup for TP2 HiCache.

中文：TP4 长上下文回放使用 6 个 tokenizer worker；TP2 HiCache 保持 SGLang 默认单 worker 启动路径，避免共享内存初始化竞态。
---
 .../single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index 40aa2d76d1..784d8d02ed 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -81,6 +81,14 @@ PARALLEL_ARGS=(
     --ep-size "$EP_SIZE"
 )
 
+# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client
+# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer
+# startup races with the TP2 HiCache shared-memory initialization path.
+TOKENIZER_ARGS=()
+if [ "$TP" -ge 4 ]; then
+    TOKENIZER_ARGS=(--tokenizer-worker-num 6)
+fi
+
 # AgentX concurrency counts live session trees rather than individual HTTP
 # requests. Leave room for subagent fan-out and avoid spending HBM on graphs
 # above the batch sizes that remain useful for this long-context workload.
@@ -122,6 +130,7 @@ SGLANG_CMD=(
     --mem-fraction-static 0.80
     --stream-interval 50
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
+    "${TOKENIZER_ARGS[@]}"
     --tokenizer-path "$MODEL"
     --reasoning-parser qwen3
     --tool-call-parser qwen3_coder

From 3ed39d3d48fccd0da133185afbf20018f3475119 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 04:09:48 -0500
Subject: [PATCH 26/32] perf(agentx): trim B200 cache-thrash tail
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Keep the measured Pareto region and validate every node-local checkpoint shard before selecting NVMe weights.\n\n中文：保留实测帕累托区间，并在选择 NVMe 权重前校验节点本地检查点的全部分片。
---
 configs/nvidia-master.yaml  | 12 ++++++++----
 runners/launch_b200-dgxc.sh | 23 ++++++++++++++++++-----
 2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index af013b6902..4cf792692e 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -8484,10 +8484,14 @@ minimaxm3-fp8-h200-vllm-agentic:
 # measured 15.3 KB/token. Against the 256k trace's 88.8k median and 148.8k
 # p75 ISL, TP2 no-offload is already GPU-KV-bound by c18; measured c20
 # HiCache improves both its throughput and latency, so offload takes over
-# after c14 rather than repeating dominated no-offload points.
+# after c14 rather than repeating dominated no-offload points. TP2 c40 is
+# capped to 25 running requests by the available HBM and matches c32 throughput
+# with 15-second median TTFT, so the HiCache arm ends at c32.
 # TP4 retains 83.9% prefix hits at c64. No-offload c66 and c68 are already
 # throughput/latency dominated, while c70 collapses to 6% hits and 96% KV
-# usage, so HiCache takes over at c64 and densely covers the boundary.
+# usage, so HiCache takes over at c64 and densely covers the boundary through
+# c76. At c80 GPU KV reaches 100% with 19-second median TTFT, and c88 falls to
+# 10% prefix hits, so the dominated upper tail is excluded.
 qwen3.5-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:v0.5.16-cu130
   model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -8501,9 +8505,9 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
     - dram-utilization: 0.80
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 60, 62, 64] }
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76, 80, 88, 96] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64, 66, 68, 70, 72, 76] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32, 40, 48] }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] }
 
 dsv4-fp4-b200-sglang-agentic-hicache:
   image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233
diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh
index 95924652d6..f3d31f6102 100644
--- a/runners/launch_b200-dgxc.sh
+++ b/runners/launch_b200-dgxc.sh
@@ -470,15 +470,28 @@ else
     salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
 
-    # Qwen3.5 FP4 is staged on local NVMe on some, but not all, DGXC nodes.
+    # Qwen3.5 FP4 may be staged on local NVMe on a DGXC node.
     # Resolve this only after Slurm assigns the node; checking on the login
-    # host would select the wrong filesystem. A config plus the safetensor
-    # index are required so a partial staging directory is never selected.
+    # host would select the wrong filesystem. Validate every shard named by
+    # the safetensor index so a partial staging directory is never selected.
     if [[ "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" ]]; then
         LOCAL_MODEL_PATH="/raid/models/Qwen3.5-397B-A17B-NVFP4"
         if srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 \
-            test -f "$LOCAL_MODEL_PATH/config.json" -a \
-                 -f "$LOCAL_MODEL_PATH/model.safetensors.index.json"; then
+            python3 - "$LOCAL_MODEL_PATH" <<'PY'; then
+import json
+from pathlib import Path
+import sys
+
+model_path = Path(sys.argv[1])
+index_path = model_path / "model.safetensors.index.json"
+if not (model_path / "config.json").is_file() or not index_path.is_file():
+    raise SystemExit(1)
+
+with index_path.open() as index_file:
+    shards = set(json.load(index_file).get("weight_map", {}).values())
+if not shards or any(not (model_path / shard).is_file() for shard in shards):
+    raise SystemExit(1)
+PY
             MODEL_PATH="$LOCAL_MODEL_PATH"
             export MODEL_PATH
             echo "Using node-local NVMe checkpoint: $MODEL_PATH"

From 3a1b5f76f474fa4efc750b10b9932bcf578d05ba Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 12:05:45 -0500
Subject: [PATCH 27/32] fix(agentx): cap Qwen trace idle gaps

Pin the AIPerf trace idle-gap branch and set the Qwen B200 replay cap to 300 seconds.
---
 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 1 +
 perf-changelog.yaml                                           | 1 +
 utils/aiperf                                                  | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
index 784d8d02ed..d2bcc15996 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh
@@ -167,6 +167,7 @@ if [ "${EVAL_ONLY:-false}" = "true" ]; then
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --trace-idle-gap-cap-seconds 300"
     REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index fdc7f7c2ea..9e4ff4bb76 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -5345,5 +5345,6 @@
   description:
     - "Add Qwen3.5-397B-A17B NVFP4 AgentX benchmark on B200 with SGLang native NEXTN MTP"
     - "Use the 256k trace dataset and golden synthetic acceptance length 3.39"
+    - "Pin AIPerf trace idle-gap support and cap per-trace idle gaps at 300 seconds"
     - "Image: lmsysorg/sglang:v0.5.16-cu130"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420
diff --git a/utils/aiperf b/utils/aiperf
index be758d6218..c1f4ea78f1 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit be758d6218268171e2957fbec9d4f557275bca2d
+Subproject commit c1f4ea78f1226cddc00ec45f553569d3b1b7c3f6

From e1ab4ff98ed8f1c62813f5c117a0e4ef50774b9a Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 12:14:31 -0500
Subject: [PATCH 28/32] chore(aiperf): pin merged trace idle cap branch

---
 utils/aiperf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/aiperf b/utils/aiperf
index c1f4ea78f1..63d8be31b6 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit c1f4ea78f1226cddc00ec45f553569d3b1b7c3f6
+Subproject commit 63d8be31b6d03f0f30a76efa8db35fa077ffefc8

From 6d21400f1bdd9954f08ba8f53c5487f0962d68be Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 13:16:15 -0500
Subject: [PATCH 29/32] chore(agentx): bump AIPerf trace idle cap fix

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 utils/aiperf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/aiperf b/utils/aiperf
index 63d8be31b6..1e5015f2c6 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit 63d8be31b6d03f0f30a76efa8db35fa077ffefc8
+Subproject commit 1e5015f2c6f5ea220f2025fe8c4c6c1534e87f77

From 4d63a16c96c2b953553dba10b0a768db4da336f4 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 13:35:04 -0500
Subject: [PATCH 30/32] fix(agentx): update AIPerf trace-cap cleanup

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 utils/aiperf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/aiperf b/utils/aiperf
index 1e5015f2c6..040766e5e0 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit 1e5015f2c6f5ea220f2025fe8c4c6c1534e87f77
+Subproject commit 040766e5e0ea1656283b6a76c870d1bd35bda092

From 920618bc66c1fa1c9d83dd340bcab09e0d87e2bb Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 14:58:13 -0500
Subject: [PATCH 31/32] chore(agentx): pin reconstruction-only idle cap

Update AIPerf after removing runtime trace idle enforcement.

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 utils/aiperf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/aiperf b/utils/aiperf
index 040766e5e0..61a29cda34 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit 040766e5e0ea1656283b6a76c870d1bd35bda092
+Subproject commit 61a29cda34b828c4624278a1ae70910d6487156d

From e040e7c0bb5449d5fe83f7f1e31265526fd7be34 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Thu, 30 Jul 2026 16:39:56 -0500
Subject: [PATCH 32/32] fix(agentx): update AIPerf warmup handoff

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
---
 utils/aiperf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/aiperf b/utils/aiperf
index 61a29cda34..b6e4ecf4ba 160000
--- a/utils/aiperf
+++ b/utils/aiperf
@@ -1 +1 @@
-Subproject commit 61a29cda34b828c4624278a1ae70910d6487156d
+Subproject commit b6e4ecf4ba87ed1ab085380828c464ea601343d2