From 929a3c201d20867d743e9d174636fa518b13b552 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 22:35:37 +0900 Subject: [PATCH 01/19] [AMD] [AGENTX] Add dsv4-fp4-mi355x-vllm-agentic-mtp (MTP variant) New MTP agentic recipe for DeepSeek-V4-Pro FP4 on MI355X vLLM, derived from the non-MTP recipe with --speculative-config method=mtp num_speculative_tokens=3. Throughput runs pin synthetic MTP acceptance to the dsv4-pro golden AL; EVAL_ONLY uses real target verification (synthetic acceptance corrupts the eval score). Carries the shared fixes: EVAL_FRAMEWORK=lm-eval, per-DP-rank max-num-seqs, DEP8 max-num-batched-tokens MNBT*2, GPU_MEM_UTIL=0.85, lmcache v0.5.2. amd-master.yaml: add dsv4-fp4-mi355x-vllm-agentic-mtp entry (TP8/DPA GPU-resident conc 48; other arms commented for later sweeps). Co-Authored-By: Claude Opus 4.6 --- .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 506 ++++++++++++++++++ configs/amd-master.yaml | 23 + 2 files changed, 529 insertions(+) create mode 100644 benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh new file mode 100644 index 0000000000..ef070385ac --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -0,0 +1,506 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using vLLM. +# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the +# agentic sweep can probe both interactivity and throughput regimes: +# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across +# all $TP GPUs in a single engine. Lower TPOT, lower batch. +# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE +# experts EP-sharded within the TP group. +# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with +# experts EP-sharded across DP ranks (per the vLLM blog recipe). +# Highest aggregate throughput at large CONC. +# +# Serving flags follow the validated MI355X recipe from +# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x +# https://github.com/SemiAnalysisAI/InferenceX/blob/main/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh +# Image is configured in amd-master.yaml. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires one of these. +# KV_OFFLOAD_BACKEND=vllm-native. +# KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOAD_BACKEND=lmcache. +# KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +GPU_COUNT=$TP +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 + exit 1 +fi +export GPU_COUNT + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi +rocm-smi + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Nightly ROCm image may be missing runtime deps; ensure they are present. +agentic_pip_install --quiet Pillow fastapi uvicorn + +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 + +# vllm-project/router expands the one HTTP backend into one logical worker per +# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's +# X-Correlation-ID is stable for every turn of a conversation; alias it to the +# router's preferred X-Session-ID header. +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_VERSION=0.1.14 + VLLM_ROUTER_POLICY=consistent_hash + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" +fi + +# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while +# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's +# store mask. 32k matches the trace-replay tuning validated for this workload. +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 + +# VLLM_PREFIX_CACHE_RETENTION_INTERVAL only applies to sliding-window/Mamba +# models; this vLLM build raises ValueError if it is set for DSv4. + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +OFFLOAD_ARGS=() + +if agentic_kv_offload_enabled; then +case "${KV_OFFLOAD_BACKEND:-}" in + vllm-native) + require_agentic_kv_offload_backend vllm-native + # ---- vLLM native config ---------------------------------------------------------- + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # MI355X nodes have ~2.7 TiB of host DRAM available for offload; + # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for + # worker RSS / page cache / slurm cgroup). + TOTAL_CPU_DRAM_PARTITION_GB="$TOTAL_CPU_DRAM_GB" + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + ) + + ;; + mooncake) + require_agentic_kv_offload_backend mooncake + # ---- Mooncake config ---------------------------------------------------------- + # Embedded mode contributes one segment per GPU rank to a shared + # distributed store, so pre-divide the aggregate host-memory budget. + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT)) + + #MOONCAKE_VERSION=0.3.11.post1 + #apt-get update && apt-get install -y libcurl4 libibverbs1 rdma-core librdmacm1 libnuma1 liburing2 + #agentic_pip_install --quiet --no-cache-dir --no-deps \ + # --force-reinstall "mooncake-transfer-engine-non-cuda==$MOONCAKE_VERSION" + + git clone https://github.com/kvcache-ai/Mooncake.git + cd Mooncake + bash dependencies.sh + mkdir build + cd build + cmake .. + make -j + sudo make install # optional, make it ready to be used by vLLM/SGLang + cd .. + cd .. + + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + export INFERENCEX_MOONCAKE_MAX_TRANSFER_BATCH_KEYS=32 + python3 "$(dirname "$0")/patch_vllm_mooncake_transfer_batches.py" + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + + sleep 10 + MOONCAKE_MASTER_PID=$! + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # ---- Lmcache config ---------------------------------------------------------- + LMCACHE_PID="" + + cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi + } + + trap cleanup_lmcache_server EXIT + + cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$ROUTER_PID" "vLLM router" + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + exit "$exit_code" + } + trap cleanup_agentic_services EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 10 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + } + { set +x; } 2>/dev/null + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + git checkout v0.5.2 + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation + cd .. + + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + TOTAL_CPU_DRAM_PARTITION_GB="$TOTAL_CPU_DRAM_GB" + # Match the B200 Kimi LMCache setup: keep a 2.5 TB semantic CPU KV + # pool, but let the external MP server own that pool so vLLM does not + # split --kv-offloading-size across TP ranks through the integrated + # LMCache backend. + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector a + # ZMQ-style host string. + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + LMCACHE_L1_SIZE_GB="${TOTAL_CPU_DRAM_PARTITION_GB}" + if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 + exit 1 + fi + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + # LMCache read locks are leases on chunks that lookup has promised + # vLLM can retrieve. The default 300s TTL is too short for this + # long-context agentic queue: TP8/conc32 can spend >300s between + # lookup and retrieve while GPU KV is saturated, which leaves the + # object present in L1 but no longer readable. Keep the 2.5 TB pool + # size unchanged and only extend the lookup-to-retrieve lease. + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-1024}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=1200 + LMCACHE_TX_MODE="lmcache_driven" + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + --supported-transfer-mode "$LMCACHE_TX_MODE" + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '${KV_OFFLOAD_BACKEND:-}' (expected: vllm-native, mooncake, lmcache)" >&2 + exit 1 + ;; +esac +fi + +# ---- LLM server config ---------------------------------------------------------- +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +# DEP8 (TP8 + DP-attention) is a high-concurrency arm tuned separately from the +# smaller DEP arm: larger prefill token budget and long-prefill chunking so +# decode latency stays bounded at high concurrency. +IS_DEP8=false +if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then + IS_DEP8=true +fi + +MODE_ARGS=() +MNBT=8192 +if [ "$EP_SIZE" -gt 1 ]; then + MODE_ARGS+=(--enable-ep-weight-filter) +fi +if [ "$DP_ATTENTION" = "true" ]; then + MODE_ARGS+=(--prefill-schedule-interval 8) + if [ "$IS_DEP8" = "true" ]; then + MODE_ARGS+=( + --max-num-batched-tokens $((MNBT * 2)) + --long-prefill-token-threshold 16384 + ) + else + MODE_ARGS+=(--max-num-batched-tokens $MNBT) + fi +fi + +# --max-num-seqs is applied PER scheduler. Under DP-attention each of the $TP +# DP ranks runs its own scheduler and the router spreads sessions across them, +# so size the per-rank cap as 2*CONC/TP (aggregate = 2*CONC). In pure-TP there +# is a single scheduler across all GPUs that sees all CONC sessions, so use +# 2*CONC directly. +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_SEQS=$((2 * CONC / TP)) +else + MAX_NUM_SEQS=$((2 * CONC)) +fi +CONTEXT_LEN=1048576 + +# MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, +# every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the +# explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N) -- one graph per +# decode batch of 1..MAX_NUM_SEQS seqs. vLLM rounds configured sizes up to +# multiples of (1+N) and dedups (adjust_cudagraph_sizes_for_spec_decode), so a +# plain 1..MAX_NUM_SEQS list would collapse to coverage of only +# MAX_NUM_SEQS/(1+N) seqs and drop the largest decode batches to eager. +NUM_SPEC_TOKENS=3 +TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) +# Throughput pins synthetic MTP acceptance to the dsv4-pro golden AL (thinking_on, +# num_speculative_tokens=3, golden_al_distribution/dsv4_mtp.yaml). The EVAL_ONLY +# accuracy run uses real target verification instead -- synthetic acceptance +# bypasses verification and corrupts the SWE-bench eval (0.0000 score). +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" +else + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 2.49}" +fi + +# These MI355X nodes have a stable ~32 GiB/GPU carveout: only ~256/288 GiB is +# free at init, independent of the KV-offload backend (observed identically on +# both the lmcache and GPU-resident kv-none arms). At +# --gpu-memory-utilization 0.95 vLLM requests 273.6 GiB and every DP worker +# hard-fails ("Free memory ... less than desired GPU memory utilization"), so +# cap utilization at 0.85 (244.8 GiB, ~11 GiB headroom). +GPU_MEM_UTIL=0.85 + +echo "Starting vllm server..." +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +sleep 180 + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --trust-remote-code + --async-scheduling + --distributed-executor-backend mp + --kv-cache-dtype fp8 + --block-size 256 + --max-model-len "$CONTEXT_LEN" + "${PARALLEL_ARGS[@]}" + "${MODE_ARGS[@]}" + --gpu-memory-utilization "$GPU_MEM_UTIL" + --moe-backend aiter + --compilation-config '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY"}' + --speculative-config "$SPEC_CONFIG" + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --enable-prefix-caching + --no-disable-hybrid-kv-cache-manager + --max-num-seqs "$MAX_NUM_SEQS" + "${OFFLOAD_ARGS[@]}" +) + +# (srok), not yet + #--attention_config.use_fp4_indexer_cache=True +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_VLLM_ROUTER" = "true" ]; then + echo "Starting native vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy "$VLLM_ROUTER_POLICY" \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 8e6bee505e..b5587022df 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1536,6 +1536,29 @@ dsv4-fp4-mi355x-vllm-agentic: # image tag, so bumping sglang is just an image tag bump here. Sweeps # DP-attention on/off and EP=8. +dsv4-fp4-mi355x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-49f31d7cee425a6d38f8c5bc76877986daf832ed + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48] } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, conc-list: [76] } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [48] } + +# DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the +# amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the +# image tag, so bumping sglang is just an image tag bump here. Sweeps +# DP-attention on/off and EP=8. + dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 model: amd/DeepSeek-R1-0528-MXFP4-v2 From 0a1c94f40d34b5b8b739bccea3e775270c1c5567 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 22:43:27 +0900 Subject: [PATCH 02/19] docs(perf-changelog): add dsv4-fp4-mi355x-vllm-agentic-mtp entry for #2381 Co-Authored-By: Claude Opus 4.6 --- perf-changelog.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b34f5aa3d4..e4901a6c52 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5137,4 +5137,12 @@ - "Remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm from the 1P1D TP1-prefill/TP8-decode recipe; retain the existing FP8 indexer KV cache configuration" - "Add a one-node 4P1D TP1-prefill/TEP4-decode concurrency-4096 point on the refreshed image while preserving the legacy 4P2D DEP2-prefill/TEP4-decode point unchanged" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2310 - + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + description: + - "Add an MTP variant of the DeepSeek-V4-Pro FP4 MI355X vLLM agentic recipe (new benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh + amd-master.yaml entry). Adds --speculative-config method=mtp num_speculative_tokens=3; throughput runs pin synthetic acceptance to the dsv4-pro golden AL (synthetic_acceptance_length=2.49) while EVAL_ONLY runs use real target verification (synthetic acceptance zeroes the eval score)." + - "Carries the shared vllm-agentic fixes: EVAL_FRAMEWORK=lm-eval (never swebench); per-DP-rank --max-num-seqs=2*CONC/TP under DP-attention (2*CONC in pure-TP); DEP8 --max-num-batched-tokens=MNBT*2 (16384) to avoid the rocBLASLt/TensileLite segfault at M=65536; --gpu-memory-utilization 0.85 (MI355X ~32 GiB/GPU carveout makes 0.95 OOM at init); lmcache v0.5.2 with chunk-size 1024 and undivided CPU pool; --block-size 256, --max-model-len 1048576, cudagraph FULL_DECODE_ONLY." + - "Initial active arm: TP8/DPA GPU-resident (kv-offloading none) conc 48; other arms commented out for later sweeps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2381 + From a1a61eb9fbd82cfa7f145c2163a0556900bcb3dd Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 23:01:54 +0900 Subject: [PATCH 03/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic-mtp: use 2*CONC max-num-seqs Override the per-DP-rank sizing for the MTP recipe; use 2*CONC unconditionally for better throughput. Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index ef070385ac..e6f706c966 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -403,11 +403,14 @@ fi # so size the per-rank cap as 2*CONC/TP (aggregate = 2*CONC). In pure-TP there # is a single scheduler across all GPUs that sees all CONC sessions, so use # 2*CONC directly. -if [ "$DP_ATTENTION" = "true" ]; then - MAX_NUM_SEQS=$((2 * CONC / TP)) -else - MAX_NUM_SEQS=$((2 * CONC)) -fi +#if [ "$DP_ATTENTION" = "true" ]; then +# MAX_NUM_SEQS=$((2 * CONC / TP)) +#else +# MAX_NUM_SEQS=$((2 * CONC)) +#fi + +# enforce to use 2xCONC for better throughput +MAX_NUM_SEQS=$((2 * CONC)) CONTEXT_LEN=1048576 # MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, From 279651b6b94ae89cea4f0cc765d9e705904b4056 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Tue, 28 Jul 2026 23:28:33 +0900 Subject: [PATCH 04/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic-mtp: add spec-decoding: mtp + vllm-router Add spec-decoding: mtp to all search-space arms so the launcher's SPEC_SUFFIX routes to dsv4_fp4_mi355x_vllm_mtp.sh (without it, SPEC_DECODING is none and the non-MTP script runs); add the vllm-router 0.1.14 to the active DP-attention arm. Fix a missing closing brace on the active-arm flow mapping. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b5587022df..df7ad782b4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1548,11 +1548,11 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48] } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } - #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, conc-list: [76] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [48] } + #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From c7c1f239e2452abcc7ef2fe061ad95c8fcfed3a2 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 00:09:03 +0900 Subject: [PATCH 05/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic-mtp: revert to per-DP-rank max-num-seqs Restore MAX_NUM_SEQS=2*CONC/TP under DP-attention (2*CONC in pure-TP). At 2*CONC (=96) the MTP cudagraph capture list grew to 512 sizes and the activation budget left only 23.36 GiB for KV cache -- below the 24.06 GiB needed for one max-len request, so engine init failed in _check_enough_kv_cache_memory. Per-rank sizing shrinks the capture/activation footprint so KV cache fits. Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index e6f706c966..ef070385ac 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -403,14 +403,11 @@ fi # so size the per-rank cap as 2*CONC/TP (aggregate = 2*CONC). In pure-TP there # is a single scheduler across all GPUs that sees all CONC sessions, so use # 2*CONC directly. -#if [ "$DP_ATTENTION" = "true" ]; then -# MAX_NUM_SEQS=$((2 * CONC / TP)) -#else -# MAX_NUM_SEQS=$((2 * CONC)) -#fi - -# enforce to use 2xCONC for better throughput -MAX_NUM_SEQS=$((2 * CONC)) +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_SEQS=$((2 * CONC / TP)) +else + MAX_NUM_SEQS=$((2 * CONC)) +fi CONTEXT_LEN=1048576 # MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, From ad522ad3f46a474475c589f0e037fd5ee9b1b6bb Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 10:35:35 +0900 Subject: [PATCH 06/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-mtp: bump gpu-mem-util 0.85 -> 0.86 for MTP KV headroom At 0.85 the ~23.9 GiB KV budget falls below the 24.06 GiB one 1M-context request needs once the MTP draft layer's per-token KV is counted, so engine init dies on the tighter eval-only relaunch. 0.86 adds ~2.6 GiB (nearly all to KV) while staying ~8 GiB under the free-mem hard-fail ceiling. Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index ef070385ac..5ab07e7956 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -433,9 +433,15 @@ fi # free at init, independent of the KV-offload backend (observed identically on # both the lmcache and GPU-resident kv-none arms). At # --gpu-memory-utilization 0.95 vLLM requests 273.6 GiB and every DP worker -# hard-fails ("Free memory ... less than desired GPU memory utilization"), so -# cap utilization at 0.85 (244.8 GiB, ~11 GiB headroom). -GPU_MEM_UTIL=0.85 +# hard-fails ("Free memory ... less than desired GPU memory utilization"). +# But 0.85 (244.8 GiB) leaves only ~23.9 GiB for KV, which is below the +# 24.06 GiB one request at max_model_len=1M needs once the MTP draft layer's +# extra per-token KV is counted -- engine init then dies with +# "available KV cache memory ... larger than ..." on the tighter (eval-only) +# relaunch. 0.86 (247.7 GiB) adds ~2.6 GiB, nearly all to KV (~26.5 GiB), so +# the KV check clears with margin while still keeping ~8 GiB free-mem headroom +# below the ~256 GiB hard-fail ceiling. +GPU_MEM_UTIL=0.86 echo "Starting vllm server..." set -x From 81a95c31064739c25de02af8bcb1a05b53d6e35f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 11:28:46 +0900 Subject: [PATCH 07/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic-mtp: expand DEP8 MTP conc sweep to [1,4,8,16,32,48,64] Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d75aac47f4..626775b89e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1552,7 +1552,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48, 64], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From 765ad4d7b4bc2401393b19ef87fb86c2a258dc47 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 11:56:18 +0900 Subject: [PATCH 08/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-mtp: clamp DP-attn max-num-seqs to >=1 Per-rank cap 2*CONC/TP floor-divides to 0 for CONC=1 on DEP8 (2/8=0), which vLLM rejects (SchedulerConfig requires max_num_seqs >= 1). Clamp to a minimum of 1 so the newly-added low-conc DEP8 arms launch. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 5ab07e7956..b632f503b9 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -402,9 +402,14 @@ fi # DP ranks runs its own scheduler and the router spreads sessions across them, # so size the per-rank cap as 2*CONC/TP (aggregate = 2*CONC). In pure-TP there # is a single scheduler across all GPUs that sees all CONC sessions, so use -# 2*CONC directly. +# 2*CONC directly. Floor-divide can yield 0 for small CONC under DP-attention +# (e.g. CONC=1, TP=8 -> 2/8=0), which vLLM rejects (SchedulerConfig requires +# max_num_seqs >= 1), so clamp the per-rank cap to a minimum of 1. if [ "$DP_ATTENTION" = "true" ]; then MAX_NUM_SEQS=$((2 * CONC / TP)) + if [ "$MAX_NUM_SEQS" -lt 1 ]; then + MAX_NUM_SEQS=1 + fi else MAX_NUM_SEQS=$((2 * CONC)) fi From f831c484d8dc14eb1f013ed3b286cc058f0a80be Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 14:04:32 +0900 Subject: [PATCH 09/19] [AMD] [AGENTX] dsv4-fp4-mi355x-vllm-agentic-mtp: drop conc 64 from DEP8 MTP sweep Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 626775b89e..902740a6ee 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1552,7 +1552,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48, 64], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From d06df9d9a9c027ff055b215815dc301331d53b4f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 15:41:05 +0900 Subject: [PATCH 10/19] [AMD][AgentX] dsv4-mtp: widen worker RPC timeout for long-context arm Long-context forward passes (~370K tokens with fp8 KV + DRAM offload) exceed vLLM's default 300s VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, killing the engine with "RPC call to sample_tokens timed out". Default it to 1200s (overridable). Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index b632f503b9..faaffa0d42 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -448,6 +448,11 @@ fi # below the ~256 GiB hard-fail ceiling. GPU_MEM_UTIL=0.86 +# Long-context forward passes (~370K tokens with fp8 KV + DRAM offload) can exceed +# vLLM's default 300s worker RPC timeout, killing the engine with +# "RPC call to sample_tokens timed out". Widen it. +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-1200}" + echo "Starting vllm server..." set -x export VLLM_ROCM_USE_AITER=1 From ff982e578c9647e4a366c4e41f3224a9d32d4f80 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 15:50:03 +0900 Subject: [PATCH 11/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: narrow DEP8 MTP sweep to conc 48 Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 902740a6ee..60cefb72fc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1552,7 +1552,8 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From 7a580b96257285f094bc6e5ffc663f1b94f36351 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 22:23:36 +0900 Subject: [PATCH 12/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: swap kv-none arm for dram mooncake/lmcache Replace the DEP8 MTP conc48 kv-offloading:none arm with two dram-offload arms (mooncake pinned to ac5ec1d, lmcache v0.5.2) and pin the Mooncake build in the agentic script to that same revision. Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 1 + configs/amd-master.yaml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index faaffa0d42..908f21e685 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -156,6 +156,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in git clone https://github.com/kvcache-ai/Mooncake.git cd Mooncake + git checkout ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999 bash dependencies.sh mkdir build cd build diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 60cefb72fc..84fef6904d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1553,7 +1553,9 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From aadd6779fd49985029c4757b1515c503c3f3488d Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 22:24:44 +0900 Subject: [PATCH 13/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: add dram vllm-native arm Add a DEP8 MTP conc48 dram-offload arm using the vllm-native KV-offload backend alongside the existing mooncake and lmcache arms. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 84fef6904d..80fb571a97 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1556,6 +1556,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From 4e7b8ad96ce7f962ebb6651b6f62c4360aee82c5 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 29 Jul 2026 22:25:17 +0900 Subject: [PATCH 14/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: fix duplicate kv-offload-backend key Remove the accidentally duplicated kv-offload-backend key on the vllm-native dram arm. Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 80fb571a97..d350556e9a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1556,7 +1556,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From cf233c26857409dc828dfaef973d7524133e65d1 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 09:00:04 +0900 Subject: [PATCH 15/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: bump mooncake/lmcache versions, raise GPU_MEM_UTIL to 0.90 Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 11 +++++------ configs/amd-master.yaml | 5 +++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 908f21e685..6fad0984c8 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -156,7 +156,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in git clone https://github.com/kvcache-ai/Mooncake.git cd Mooncake - git checkout ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999 + git checkout v0.3.12 bash dependencies.sh mkdir build cd build @@ -167,8 +167,6 @@ case "${KV_OFFLOAD_BACKEND:-}" in cd .. python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - export INFERENCEX_MOONCAKE_MAX_TRANSFER_BATCH_KEYS=32 - python3 "$(dirname "$0")/patch_vllm_mooncake_transfer_batches.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -292,7 +290,8 @@ EOF git clone https://github.com/LMCache/LMCache.git cd LMCache - git checkout v0.5.2 + #git checkout v0.5.2 + git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 pip install -r requirements/build.txt CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation cd .. @@ -446,8 +445,8 @@ fi # "available KV cache memory ... larger than ..." on the tighter (eval-only) # relaunch. 0.86 (247.7 GiB) adds ~2.6 GiB, nearly all to KV (~26.5 GiB), so # the KV check clears with margin while still keeping ~8 GiB free-mem headroom -# below the ~256 GiB hard-fail ceiling. -GPU_MEM_UTIL=0.86 +# below the ~256 GiB hard-fail ceiling. Additional 0.9 for mooncake headroom +GPU_MEM_UTIL=0.90 # Long-context forward passes (~370K tokens with fp8 KV + DRAM offload) can exceed # vLLM's default 300s worker RPC timeout, killing the engine with diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9b6c781ae4..4004009a04 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1554,8 +1554,9 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "ac5ec1dce9ec2619a6f5f96c6f63cbbfb883d999" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "v0.3.12" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the From 9dd8884a00b1d35cccc6beb1bfce40e4c2e116ce Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 09:31:24 +0900 Subject: [PATCH 16/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: lower dram-utilization to 0.60 Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4004009a04..d007e3f928 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1546,7 +1546,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.60 search-space: #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } From c768e7b6cb751ba1fe6d8fbd1610df611e861100 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 09:36:24 +0900 Subject: [PATCH 17/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: disable vllm-native dram arm Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d007e3f928..e15de15467 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1557,7 +1557,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "v0.3.12" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the From 028074ed784e7504eba983c72a4b125ed1ce3c92 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 12:04:42 +0900 Subject: [PATCH 18/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: revert lmcache to v0.5.2 and set DP-attn max-num-seqs to CONC Co-Authored-By: Claude Opus 4.6 --- .../single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 13 +++++-------- configs/amd-master.yaml | 3 +-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 6fad0984c8..fa0aaf3107 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -290,8 +290,7 @@ EOF git clone https://github.com/LMCache/LMCache.git cd LMCache - #git checkout v0.5.2 - git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 + git checkout v0.5.2 pip install -r requirements/build.txt CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation cd .. @@ -400,13 +399,11 @@ fi # --max-num-seqs is applied PER scheduler. Under DP-attention each of the $TP # DP ranks runs its own scheduler and the router spreads sessions across them, -# so size the per-rank cap as 2*CONC/TP (aggregate = 2*CONC). In pure-TP there -# is a single scheduler across all GPUs that sees all CONC sessions, so use -# 2*CONC directly. Floor-divide can yield 0 for small CONC under DP-attention -# (e.g. CONC=1, TP=8 -> 2/8=0), which vLLM rejects (SchedulerConfig requires -# max_num_seqs >= 1), so clamp the per-rank cap to a minimum of 1. +# but size the per-rank cap as CONC directly (not CONC/TP) so each rank can +# hold the full session count. In pure-TP there is a single scheduler across +# all GPUs that sees all CONC sessions, so use 2*CONC directly. if [ "$DP_ATTENTION" = "true" ]; then - MAX_NUM_SEQS=$((2 * CONC / TP)) + MAX_NUM_SEQS=$((CONC)) if [ "$MAX_NUM_SEQS" -lt 1 ]; then MAX_NUM_SEQS=1 fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e15de15467..5c461b821b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1555,8 +1555,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "v0.3.12" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the From 41a3d5ac07157bf12e7acc315a344615288ef496 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 30 Jul 2026 16:51:38 +0900 Subject: [PATCH 19/19] [AMD][AgentX] dsv4-fp4-mi355x-vllm-agentic-mtp: bump vllm image and switch to kvnone pure-TP sweep Co-Authored-By: Claude Opus 4.6 --- configs/amd-master.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 5c461b821b..f59441c4eb 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1537,7 +1537,7 @@ dsv4-fp4-mi355x-vllm-agentic: # DP-attention on/off and EP=8. dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-49f31d7cee425a6d38f8c5bc76877986daf832ed + image: vllm/vllm-openai-rocm:nightly-b88916617d3d2249bff0dae5cecb6b727c980a20 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1548,15 +1548,11 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } - #- { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, spec-decoding: mtp, conc-list: [32, 40, 48] } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [76] } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 48], router: { name: vllm-router, version: "0.1.14" } } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "v0.3.12" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } - #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "v0.3.12" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + #- { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "v0.5.2" }, spec-decoding: mtp, conc-list: [48], router: { name: vllm-router, version: "0.1.14" } } + # DP+TP 160.9 GiB + # TP 134.9 GiB + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the