Add DSR1-0528 FP8 MI300X vLLM PD-disaggregation

.github/configs/amd-master.yaml

@@ -166,6 +166,60 @@ dsr1-fp8-mi355x-sglang-mtp:
       search-space:
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp }
 
+dsr1-fp8-mi300x-vllm-disagg:
+  image: vllm-mori-pd:milestone4-aiterwheel
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: mi300x-disagg
+  precision: fp8
+  framework: vllm-disagg
+  multinode: true
+  disagg: true
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      - spec-decoding: "mtp"
+        conc-list: [ 8, 16, 32, 64, 128, 256, 512 ]
+        prefill:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=2"
+          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
+        decode:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=2"
+          - "DECODE_MTP_SIZE=3"
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - spec-decoding: "mtp"
+        conc-list: [ 8, 16, 32, 64, 128, 256, 512 ]
+        prefill:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=2"
+          - "VLLM_MORIIO_CONNECTOR_READ_MODE=1"
+        decode:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=2"
+          - "DECODE_MTP_SIZE=3"
+
 qwen3.5-bf16-mi355x-sglang:
   image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517
   model: Qwen/Qwen3.5-397B-A17B

.gitignore

@@ -1,3 +1,5 @@
 **/__pycache__/**
 **/.coverage
 experimental/multiturn/vllm_benchmark/results/
+.venv/
+AMDSOW_CLUSTER_NOTES.md

benchmarks/multi_node/amd_utils/bench.sh

@@ -78,6 +78,9 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do
     extra_flags=""
     if [[ "$ENGINE" == "vllm-disagg" ]]; then
         extra_flags="--trust-remote-code --tokenizer $MODEL_PATH"
+        if [ "$IS_MTP" = "true" ]; then
+            extra_flags="$extra_flags --use-chat-template"
+        fi
     else
         if [ "$IS_MTP" = "true" ]; then
             extra_flags="--use-chat-template"

benchmarks/multi_node/amd_utils/env.sh

@@ -43,7 +43,18 @@ fi
 
 set +x
 
-export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES}
+# Pin RCCL/NCCL to the data-plane RDMA devices in explicit-match form.
+# Our NCCL 2.27 / RCCL 7.2 stack silently ignores the exclude/non-explicit form,
+# so every HCA name must carry its own leading '=' (e.g. '=rocep28s0,=rocep62s0,...').
+# This matches scripts/pd/lib_shared.sh in the AMDSOW delivery.
+_nccl_ib_hca=""
+IFS=',' read -ra _ib_arr <<< "$IBDEVICES"
+for _dev in "${_ib_arr[@]}"; do
+    [[ -n "$_dev" ]] && _nccl_ib_hca+="=${_dev},"
+done
+_nccl_ib_hca="${_nccl_ib_hca%,}"
+export NCCL_IB_HCA=${NCCL_IB_HCA:-$_nccl_ib_hca}
+export MORI_RDMA_DEVICES=${MORI_RDMA_DEVICES:-$IBDEVICES}
 
 # =============================================================================
 # Engine-specific environment
@@ -110,6 +121,11 @@ $1 == "DSCP" && $2 == ":" && $NF == p {
         elif [[ $NODENAME == mia1* ]]; then
             export UCX_IB_TRAFFIC_CLASS=104
             echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME"
+        elif [[ -n "${MORI_RDMA_TC:-}" ]]; then
+            # AMDSOW MI300X Thor fallback: align UCX RoCEv2 TC with MORI_RDMA_TC
+            # when the hostname does not match a known InfX pattern.
+            export UCX_IB_TRAFFIC_CLASS=$MORI_RDMA_TC
+            echo "[INFO] Falling back to UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from MORI_RDMA_TC"
         else
             echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration."
         fi

benchmarks/multi_node/amd_utils/job.slurm

@@ -392,6 +392,10 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
     DOCKER_ENV_ENGINE=(
         -e VLLM_WS_PATH=${WS_PATH}
         -e MODEL_PATH=$DOCKER_MODEL_PATH
+        -e IBDEVICES=${IBDEVICES}
+        -e MORI_RDMA_DEVICES=${MORI_RDMA_DEVICES:-$IBDEVICES}
+        -e MORI_RDMA_TC=${MORI_RDMA_TC:-104}
+        -e MORI_KV_EXTRA_CONFIG_JSON=${MORI_KV_EXTRA_CONFIG_JSON:-}
         -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma
         -e UCX_SOCKADDR_TLS_PRIORITY=tcp
         -e UCX_MEMTYPE_CACHE=y
@@ -441,7 +445,9 @@ echo \"Rank \$SLURM_PROCID on \$(hostname)\"
 eval \"\$DOCKER_CMD_DETECT\"
 echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\"
 
-# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack)
+# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack).
+# Our vllm-mori-pd image is self-contained with bnxt_re ABI-8 libs; do NOT mount host RDMA
+# libraries for vllm-disagg to avoid ABI/version mismatch.
 RDMA_MOUNTS=()
 if [[ "$ENGINE" == "atom-disagg" ]]; then
 

benchmarks/multi_node/amd_utils/models_vllm.yaml

@@ -42,3 +42,10 @@ gpt-oss-120b:
   prefill_flags: "--tensor-parallel-size 8"
   decode_flags: "--tensor-parallel-size 8"
   env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_TRITON_BF16_GEMM=0 VLLM_USE_AITER_UNIFIED_ATTENTION=1 VLLM_ROCM_USE_AITER_MHA=0 ROCM_TRITON_MOE_PRESHUFFLE_SCALES=0"
+
+DeepSeek-R1-0528:
+  # DSR1-0528 FP8 MI300X vLLM PD-disaggregation (AMDSOW Milestone4).
+  # Tensor-parallel size is appended by server_vllm.sh from PREFILL_TP_SIZE / DECODE_TP_SIZE.
+  prefill_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 64 --performance-mode throughput --no-enable-chunked-prefill --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"pass_config\":{\"fuse_rope_kvcache\":true,\"fuse_rope_kvcache_cat_mla\":true,\"enable_sp\":true,\"fuse_gemm_comms\":true,\"fuse_act_padding\":true}}' --no-enable-prefix-caching --max-model-len 32768 --max-num-batched-tokens 32768 --gpu-memory-utilization 0.80"
+  decode_flags: "--dtype auto --trust-remote-code --distributed-executor-backend mp --kv-cache-dtype fp8_e4m3 --block-size 64 --performance-mode throughput --no-enable-chunked-prefill --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"pass_config\":{\"fuse_rope_kvcache\":true,\"fuse_rope_kvcache_cat_mla\":true,\"enable_sp\":true,\"fuse_gemm_comms\":true,\"fuse_act_padding\":true}}' --no-enable-prefix-caching --max-model-len 32768 --max-num-batched-tokens 32768 --gpu-memory-utilization 0.80 --speculative-config '{\"method\":\"deepseek_mtp\",\"num_speculative_tokens\":3}'"
+  env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_LINEAR=True VLLM_ROCM_USE_AITER_MLA=True VLLM_ROCM_USE_AITER_MOE=True VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=1 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 VLLM_MORIIO_CONNECTOR_READ_MODE=true VLLM_MOE_DP_CHUNK_SIZE=1024 AITER_ENABLE_VSKIP=0 AITER_ONLINE_TUNE=0 AITER_BYPASS_TUNE_CONFIG=1 MORI_EP_LAUNCH_CONFIG_MODE=AUTO MORI_SHMEM_HEAP_SIZE=24G MORI_SHMEM_MODE=ISOLATION MORI_APP_LOG_LEVEL=INFO MORI_IO_QP_MAX_SEND_WR=8192 MORI_IO_QP_MAX_CQE=16384 MORI_IO_QP_MAX_SGE=4 MORI_RDMA_TC=104 MORI_IB_PATH_MTU=1024 NCCL_IB_DISABLE=0 MORI_ALLOW_FULL_CUDAGRAPH=1"

benchmarks/multi_node/amd_utils/server_vllm.sh

@@ -52,8 +52,10 @@ MODEL_PATH="${MODEL_PATH:-${MODEL_DIR}/${MODEL_NAME}}"
 source $WS_PATH/env.sh
 
 host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}')
-# RDMA IP for Nixl KV transfer (prefer 192.168.x.x subnet if available)
-rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1)
+# RDMA IP for Nixl KV transfer: prefer Thor data-plane subnet 172.29.x.x, then 192.168.x.x,
+# then fall back to the default-gateway interface IP.
+rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^172\.29\.' | head -1)
+rdma_ip="${rdma_ip:-$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1)}"
 rdma_ip="${rdma_ip:-$host_ip}"
 host_name=$(hostname)
 
@@ -217,6 +219,33 @@ echo "Decode  node IPs: ${DECODE_ARGS}"
 # MoRI-IO proxy ZMQ registration port (must match vllm-router --vllm-discovery-address)
 PROXY_PING_PORT="${PROXY_PING_PORT:-36367}"
 
+# MoRIIO connector extra config: optional caller overrides merged with the
+# standard proxy endpoints. Prefer only proven keys; unknown keys are silently
+# ignored by the connector.
+MORI_KV_EXTRA_CONFIG_JSON="${MORI_KV_EXTRA_CONFIG_JSON:-{\"read_mode\":true,\"allow_full_cudagraph\":true}}"
+
+KV_TRANSFER_CONFIG_BASE=$(python3 -c "
+import json, os
+extra = json.loads(os.environ.get('MORI_KV_EXTRA_CONFIG_JSON', '{}'))
+print(json.dumps(extra, separators=(',', ':')))
+")
+
+# Prefill nodes are kv_producer; decode nodes are kv_consumer. The proxy IP,
+# ping port, and HTTP port are always nested inside kv_connector_extra_config.
+build_kv_config() {
+    local role="$1"
+    python3 -c "
+import json
+extra = json.loads('''${KV_TRANSFER_CONFIG_BASE}''')
+extra.update({'proxy_ip': '${NODE0_ADDR}', 'proxy_ping_port': '${PROXY_PING_PORT}', 'http_port': '${SERVER_PORT}'})
+cfg = {'kv_connector': 'MoRIIOConnector', 'kv_role': '${role}', 'kv_connector_extra_config': extra}
+print(json.dumps(cfg, separators=(',', ':')))
+"
+}
+
+KV_TRANSFER_CONFIG_PRODUCER=$(build_kv_config kv_producer)
+KV_TRANSFER_CONFIG_CONSUMER=$(build_kv_config kv_consumer)
+
 # vLLM runtime environment (static vars moved to env.sh; these depend on per-node state)
 setup_vllm_env() {
     export VLLM_NIXL_SIDE_CHANNEL_HOST=${rdma_ip}
@@ -256,7 +285,7 @@ if [ "$NODE_RANK" -eq 0 ]; then
         --served-model-name ${SERVED_MODEL} \
         --port $SERVER_PORT \
         --trust-remote-code \
-        --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\"}}' \
+        --kv-transfer-config "${KV_TRANSFER_CONFIG_PRODUCER}" \
         ${PREFILL_SERVER_CONFIG}"
 
     if [[ "$DRY_RUN" -eq 1 ]]; then
@@ -301,6 +330,12 @@ if [ "$NODE_RANK" -eq 0 ]; then
     echo "Benchmarking on ${host_name}:${host_ip}"
     cd $WS_PATH
 
+    if [[ "${DECODE_MTP_SIZE:-0}" -gt 0 ]]; then
+        export IS_MTP=true
+    else
+        export IS_MTP=false
+    fi
+
     export ROUTER_PORT=$ROUTER_PORT
     BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((GPUS_PER_NODE*xP)) $((GPUS_PER_NODE*yD)) \
         $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \
@@ -410,7 +445,6 @@ if [ "$NODE_RANK" -eq 0 ]; then
         echo "ERROR: eval failed; exiting node-0 with rc=1"
         exit 1
     fi
-
 elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then
     echo "${host_name}:${host_ip} is Additional Prefill Node (Model: ${MODEL_NAME})"
     echo "Using prefill config: $PREFILL_SERVER_CONFIG"
@@ -422,7 +456,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then
         --served-model-name ${SERVED_MODEL} \
         --port $SERVER_PORT \
         --trust-remote-code \
-        --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\"}}' \
+        --kv-transfer-config "${KV_TRANSFER_CONFIG_PRODUCER}" \
         ${PREFILL_SERVER_CONFIG}"
 
     if [[ "$DRY_RUN" -eq 1 ]]; then
@@ -478,7 +512,7 @@ else
         --served-model-name ${SERVED_MODEL} \
         --port $SERVER_PORT \
         --trust-remote-code \
-        --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\"}}' \
+        --kv-transfer-config "${KV_TRANSFER_CONFIG_CONSUMER}" \
         ${DECODE_SERVER_CONFIG}"
 
     if [[ "$DRY_RUN" -eq 1 ]]; then

benchmarks/multi_node/dsr1_fp8_mi300x_vllm-disagg.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# DSR1-0528 FP8 MI300X vLLM PD-disaggregation recipe.
+# Mirrors kimik2.5_fp4_mi355x_vllm-disagg.sh; model-specific flags live in
+# amd_utils/models_vllm.yaml under the DeepSeek-R1-0528 entry.
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    SPEC_DECODING \
+    MODEL_PATH \
+    PREFILL_NUM_WORKERS \
+    PREFILL_TP \
+    PREFILL_EP \
+    PREFILL_DP_ATTN \
+    DECODE_NUM_WORKERS \
+    DECODE_TP \
+    DECODE_EP \
+    DECODE_DP_ATTN \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO \
+    FRAMEWORK
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1
+
+export TIME_LIMIT="08:00:00"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Same EP/DP booleans as other disagg recipes → amd_utils/submit.sh
+if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then
+    export PREFILL_ENABLE_EP=false
+else
+    export PREFILL_ENABLE_EP=true
+fi
+
+if [[ "$PREFILL_DP_ATTN" == "true" ]]; then
+    export PREFILL_ENABLE_DP=true
+else
+    export PREFILL_ENABLE_DP=false
+fi
+
+if [[ "${DECODE_EP:-1}" -eq 1 ]]; then
+    export DECODE_ENABLE_EP=false
+else
+    export DECODE_ENABLE_EP=true
+fi
+
+if [[ "$DECODE_DP_ATTN" == "true" ]]; then
+    export DECODE_ENABLE_DP=true
+else
+    export DECODE_ENABLE_DP=false
+fi
+
+# Parameter order matches SGLang disagg submit.sh; arg 16 is optional NODELIST.
+JOB_ID=$(bash ./submit.sh $PREFILL_NODES \
+    $PREFILL_NUM_WORKERS \
+    $DECODE_NODES \
+    $DECODE_NUM_WORKERS \
+    $ISL $OSL "${CONC_LIST// /x}" inf \
+    ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \
+    ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \
+    ${PREFILL_TP} ${DECODE_TP} \
+    ${RANDOM_RANGE_RATIO} \
+    "${NODELIST:-}")
+
+if [[ $? -ne 0 ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

perf-changelog.yaml

@@ -4153,3 +4153,11 @@
     - "Run the PR #1891 MiniMax-M3 MXFP8 B300 Dynamo-vLLM recipe set on top of current main."
     - "Uses the vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image and the TEP4/TEP8 8k1k topologies not covered by PR #1890."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1891
+- config-keys:
+    - dsr1-fp8-mi300x-vllm-disagg
+  description:
+    - "Add DSR1-0528 FP8 MI300X vLLM PD-disaggregation configuration"
+    - "Image: vllm-mori-pd:milestone4-aiterwheel"
+    - "Topology: 2P2D TP8 with MTP3 speculative decoding"
+    - "Runner: mi300x-disagg; framework: vllm-disagg"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX