diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 34fba860f4..1327274399 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -310,14 +310,15 @@ jobs: dp-attn: ${{ matrix.config.dp-attn }} conc: ${{ matrix.config.conc }} kv-offloading: ${{ matrix.config.kv-offloading }} - kv-offload-backend: ${{ matrix.config.kv-offload-backend }} + kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} + kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} agentx-fast: ${{ inputs.agentx-fast }} isl: '0' osl: '0' max-model-len: '0' - spec-decoding: 'none' + spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ 'false' }} run-eval: true eval-only: true diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x.sh new file mode 100755 index 0000000000..15f3235820 --- /dev/null +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x.sh @@ -0,0 +1,438 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Kimi-K3 MXFP4 on MI355X / MI350X (gfx950) +# using vLLM. +# +# The server command is the AMD reference `vllm serve` for this model, i.e. the +# upstream vLLM recipe's amd block (vllm-project/recipes, +# models/moonshotai/Kimi-K3.yaml, date_updated 2026-07-25) as run in practice: +# +# --trust-remote-code --moe-backend auto --tensor-parallel-size 8 +# --load-format auto --gpu-memory-utilization 0.95 --mm-encoder-tp-mode data +# --max-num-seqs 128 --max-num-batched-tokens 4096 --enable-auto-tool-choice +# --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 +# +# with env VLLM_ROCM_USE_AITER=1 SAFETENSORS_FAST_GPU=1 AITER_SITUV2_A8W4=1 +# AITER_BF16_FP8_MOE_BOUND=0 VLLM_USE_BREAKABLE_CUDAGRAPH=0. +# +# K3 is a 2.8T-parameter natively-multimodal MoE (896 routed experts, 16/token +# plus shared) on Kimi Delta Attention, gated MLA and Attention Residuals, with +# a 1M-token native context. +# +# TP=8 ONLY. The MXFP4 checkpoint is 1.561 TB decimal (1.420 TiB, 96 +# safetensors), ~195 GB/GPU across 8 GPUs of the 288 GB part; TP=4 would need +# ~390 GB/GPU and cannot load. Upstream strategy_min_gpus agrees (single_node_tp +# and multi_node_tep both 8, DEP 16+), which is why there is no DP-attention arm. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, +# EP_SIZE +# +# GPU-only. The single shipped key +# (kimik3-fp4-mi355x-vllm-agentic-dspark) sets kv-offloading: none, and this +# recipe rejects a row that asks for a host tier rather than silently ignoring +# it. Mooncake in particular stays unsupported: the upstream recipe marks both +# kv_store_distributed_mooncake and kv_store_centralized_mooncake as +# `unsupported` on every hardware target for this model. +# +# Perf-search knobs. Each defaults to the reference command's value, so an +# otherwise-unset run reproduces the reference exactly: +# GPU_MEM_UTIL 0.95 (reference) +# MAX_NUM_SEQS 128 (reference) +# MAX_NUM_BATCHED_TOKENS 4096 (reference) +# AITER_A8W4 1 (reference; 0 = aiter a16w4 MoE path) +# LANGUAGE_MODEL_ONLY false (reference loads the vision tower) +# KV_CACHE_DTYPE fp8 (default for every arm; =auto for a bf16 A/B) +# KV_BLOCK_SIZE unset (unset -> vLLM sizes the page; 128 under fp8) +# MAX_MODEL_LEN unset (unset -> vLLM derives K3's 1M context) +# SPEC_DECODE false (enabled by the _mtp DSpark wrapper) +# ENFORCE_EAGER false (reference; DSpark wrapper keeps it false) + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# Force the eval framework to lm-eval, matching minimaxm3_fp4_mi355x.sh. +# run_eval derives its default as swebench for agentic scenarios +# (scenario_default=swebench when IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but +# EVAL_FRAMEWORK takes precedence over that default +# (benchmark_lib.sh:1471 framework=${EVAL_FRAMEWORK:-...}). Left overridable so +# the SWE-bench path below can still be selected with EVAL_FRAMEWORK=swebench. +export EVAL_FRAMEWORK="${EVAL_FRAMEWORK:-lm-eval}" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [ "$TP" -ne 8 ]; then + echo "Error: Kimi-K3 MXFP4 is a 1.56 TB checkpoint and only fits at TP=8 on" >&2 + echo " 288 GB gfx950 parts (~195 GB/GPU). Got TP=$TP." >&2 + exit 1 +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. The +# 1.56 TB checkpoint is normally pre-staged, so these calls are a no-op there. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +# kimik3* is on resolve_trace_source's unfiltered allowlist (benchmark_lib.sh), +# so this replays the full 062126 v7 corpus rather than the 256k-capped variant. +resolve_trace_source +install_agentic_deps + +# ---- Reference env block ---------------------------------------------------- +export VLLM_ROCM_USE_AITER=1 +export SAFETENSORS_FAST_GPU=1 +# AITER a8w4 MoE path for the MXFP4-weight / MXFP8-activation QAT checkpoint. +# Upstream: "set AITER_SITUV2_A8W4 to 0 along with AITER master flag to use +# aiter a16w4 MoE path. Set it to 1 to use aiter a8w4 MoE path." Swept. +export AITER_SITUV2_A8W4="${AITER_A8W4:-1}" +export AITER_BF16_FP8_MOE_BOUND=0 +# REQUIRED on ROCm per the upstream recipe: the build auto-enables this to 1. +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 + +# Workaround for MEC FW <177 RCCL memory reclaim issue (shared with the other +# gfx950 recipes in this tree). +mec_version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') +if [[ "$mec_version" == "" || ${mec_version:-0} -lt 177 ]]; then + export HSA_NO_SCRATCH_RECLAIM=1 +fi + +# 2.8T of weights off a shared/NFS mount takes far longer than the default. +export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-7200}" + +# Long agentic turns against a 1M context: keep the client from timing out +# mid-request while the server is prefill-bound. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" + +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# ---- KV cache --------------------------------------------------------------- +# This recipe ships a single GPU-only arm (kimik3-fp4-mi355x-vllm-agentic-dspark, +# kv-offloading: none), so there is no host-DRAM tier here. The LMCache / +# SimpleCPUOffload / OffloadingConnector plumbing that the bring-up sweeps used +# lives in this branch's history; re-add it with the arm that needs it rather +# than carrying dead connectors. Fail loudly instead of silently ignoring a +# matrix row that asks for one. +if agentic_kv_offload_enabled; then + echo "Error: KV offloading is not wired up for Kimi-K3 on MI355X." >&2 + echo " This recipe ships the GPU-only DSpark arm; got" >&2 + echo " KV_OFFLOADING='${KV_OFFLOADING:-}' KV_OFFLOAD_BACKEND='${KV_OFFLOAD_BACKEND:-}'." >&2 + exit 1 +fi + +# fp8 KV is the DEFAULT for this model. +# +# It is the first thing that made the trace run well: run 30442578333 (c8, +# kvnone, 3600s) scored 32.59 output tok/s / 696 total tok/s/GPU at TTFT 2.6s, +# against 19.17 / 535 / 69.1s for the bf16 cell in the same window. K3 costs +# ~217 KiB KV/token -- 3x K2.7 -- so halving the KV element size is the single +# largest lever available, exactly as in [[fp8-kv-cache-mandatory]] for +# kimik2.7. +# +# Set here, ahead of the serve command, because the mla_gluon patch further +# down gates on KV_CACHE_DTYPE=fp8 and must see it already set. The DSpark +# wrapper pins KV_CACHE_DTYPE=auto to match the upstream AMD reproducer, so +# this default only applies to stand-alone runs. +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" + +# ---- Parallelism ------------------------------------------------------------ +# TP8 or TEP8. No DP-attention arm: upstream strategy_min_gpus.multi_node_dep is +# 16, so DEP is not a single-node strategy for this model. +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +# ---- Multimodal vs text-only ------------------------------------------------ +# The reference command loads the vision tower and passes --mm-encoder-tp-mode +# data, so that is the default here. --language-model-only is an upstream +# opt_in_feature ("skip the vision encoder for text-only workloads") and the +# agentic corpus never sends an image, so it is a swept axis. +# +# Note this build's help describes the flag more narrowly than upstream's +# phrasing: "disables all multimodal inputs by setting all modality limits to 0. +# Equivalent to setting --limit-mm-per-prompt to 0 for every modality" -- input +# gating, which does not by itself guarantee the vision tower goes unloaded. +# Whether it returns HBM to the KV pool is measured by comparing +# "model weights take N GiB" in server.log across both settings. Upstream marks +# it mutually exclusive with encoder parallelism, so --mm-encoder-tp-mode is +# only passed when multimodal inputs are enabled. +LANGUAGE_MODEL_ONLY="${LANGUAGE_MODEL_ONLY:-false}" +MM_ARGS=(--mm-encoder-tp-mode data) +if [ "$LANGUAGE_MODEL_ONLY" = "true" ]; then + MM_ARGS=(--language-model-only) +fi + +# ---- Optional axes ---------------------------------------------------------- +# Only emitted when set away from the reference, so the default command line is +# byte-for-byte the reference one. +# +# fp8 KV halves bytes/token in the pool, which moves the KV-capacity wall itself +# rather than just adding headroom -- the dominant effect on a prefill-heavy 1M +# context corpus. Not on by default because K3's KV geometry is HYBRID (Kimi +# Delta Attention state + gated-MLA latent) and fp8 across both spec types is +# unconfirmed on this build. +KV_CACHE_DTYPE_ARGS=() +if [ -n "${KV_CACHE_DTYPE:-}" ] && [ "${KV_CACHE_DTYPE}" != "auto" ]; then + KV_CACHE_DTYPE_ARGS=(--kv-cache-dtype "$KV_CACHE_DTYPE") +fi + +# INERT ON K3 -- kept only so the override stays discoverable. K3 is hybrid, and +# vLLM raises the attention block size until the attention page is at least the +# mamba/KDA page (interface.py:911, "Setting attention block size to N tokens to +# ensure that attention page size is >= mamba page size"). Measured: bf16 -> 768 +# tokens, fp8 -> 1536. --block-size 64 was silently overridden to 1536 in run +# 30505656455, so this knob cannot move the page on this model; only +# kv_cache_dtype can. +# +# That also explains the connector's identical 14122 CPU blocks across dtypes: +# fp8 halves bytes/token but the page doubles in tokens, so page BYTES are +# unchanged and manager.py:199 derives the same pool. +KV_BLOCK_SIZE_ARGS=() +if [ -n "${KV_BLOCK_SIZE:-}" ] && [ "${KV_BLOCK_SIZE}" != "0" ]; then + KV_BLOCK_SIZE_ARGS=(--block-size "$KV_BLOCK_SIZE") +fi + +# Left unset by default so vLLM derives K3's native 1M context, which is what +# the unfiltered corpus needs. Set explicitly only to test truncation effects. +MAX_MODEL_LEN_ARGS=() +if [ -n "${MAX_MODEL_LEN:-}" ] && [ "${MAX_MODEL_LEN}" != "0" ]; then + MAX_MODEL_LEN_ARGS=(--max-model-len "$MAX_MODEL_LEN") +fi + +EAGER_ARGS=() +if [ "${ENFORCE_EAGER:-false}" = "true" ]; then + EAGER_ARGS=(--enforce-eager) +fi + +# The reference command passes neither --enable-prefix-caching nor +# --no-enable-prefix-caching, and this build's default is None (vLLM decides +# internally), so by default we pass nothing and stay aligned. Two reasons this +# is a knob rather than a hardcode: agentic trace replay exists to exercise +# large shared prefixes, so the resolved value must be confirmed from +# server.log; and K3 is hybrid (Kimi Delta Attention + gated MLA), where block +# and hash sizes only align with prefix caching on -- an omission has been +# reported to trip "tokens_per_block not divisible by tokens_per_hash" at load. +# Set PREFIX_CACHING=true/false to force it either way. +# ON by default for a stand-alone run. This trace is built around large shared +# prefixes -- theoretical prefix-cache hit is 98.1%, and a live GPU-only cell +# measured 92.8% server-side -- so a run with reuse disabled is not measuring +# the workload. Reuse also costs essentially no KV (1,414,660 vs 1,420,824 +# tokens) and improves ITL (484 vs 577 ms). The DSpark wrapper overrides this +# to PREFIX_CACHING=auto, which passes neither flag and matches the upstream +# AMD reproducer exactly. +# +# It was briefly defaulted off for kvnone after two cells (c2/g19, c4/g17, +# run 30412966635) died in warmup with it on, one dump naming the Mamba +# block-zeroing path (new_block_ids_to_zero=[1615] at 327,936 computed +# tokens). That was the wrong call on the evidence: c1 in the SAME run, same +# flag, cleared warmup with 0 errors and profiled 47 minutes clean before +# dying only to `srun: error: Node failure on mia1-p01-g16`. The arm plainly +# runs with the flag on, and the cluster was throwing three different +# failures across three nodes that week. Reverted. +# +# Note vLLM resolves the flag's default to False for this model, so ON must be +# passed explicitly. PREFIX_CACHING=false forces it off for a deliberate A/B. +case "${PREFIX_CACHING:-true}" in + true) + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + ;; + false) + PREFIX_CACHE_ARGS=(--no-enable-prefix-caching) + ;; + auto) + # Match the upstream AMD command by letting vLLM resolve its default. + PREFIX_CACHE_ARGS=() + ;; + *) + echo "Error: PREFIX_CACHING must be true, false, or auto." >&2 + exit 1 + ;; +esac + +# The upstream DSpark config pins "attention_backend": "FLASHINFER_MLA", which +# is CUDA-only and cannot be used verbatim on gfx950; SPEC_ATTN_BACKEND +# overrides it. Use real block rejection for both throughput and evaluation so +# this path is the exact AMD reproducer and exercises target-model verification. +SPEC_ARGS=() +if [ "${SPEC_DECODE:-false}" = "true" ]; then + SPEC_DRAFT_MODEL="${SPEC_DRAFT_MODEL:-Inferact/Kimi-K3-DSpark}" + SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-7}" + SPEC_ATTN_BACKEND="${SPEC_ATTN_BACKEND:-TRITON_MLA}" + SPEC_ARGS=( + --speculative-config + "{\"model\":\"$SPEC_DRAFT_MODEL\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"$SPEC_ATTN_BACKEND\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"block\"}" + ) +fi + +# ---- Eval-only path ----------------------------------------------------------- +# Mirrors the kimik2.7 agentic EVAL v2 configuration that scored on SWE-bench +# (run 30258968315). Two things are needed beyond the throughput config, and +# both are gated on EVAL_ONLY so the measured serving config is untouched. +EVAL_SERVE_ARGS=() +if [ "${EVAL_ONLY:-false}" = "true" ]; then + # The kimi_k3 tool-call and reasoning parsers are already passed + # unconditionally by this recipe (they are part of the AMD reference + # command), unlike kimik2.7 where they had to be added for eval. + # + # With tool calls flowing, vLLM builds a grammar for them and the default + # backend `auto` resolves to xgrammar, which rejects Kimi's tool-call tokens + # ("Failed to advance FSM" -> HTTP 500 -> empty patches -> a near-zero + # score). Move off xgrammar. llguidance is the guidance backend's runtime + # and is not in the ROCm image, so install it on demand. + SO_BACKEND="${STRUCTURED_OUTPUTS_BACKEND:-guidance}" + if [ "$SO_BACKEND" != "auto" ]; then + python3 -c 'import llguidance' 2>/dev/null || pip install --quiet llguidance || true + if python3 -c 'import llguidance' 2>/dev/null; then + # `vllm serve --help` lists config SECTIONS, not flags; the flag + # names only appear under --help=all. + VLLM_SERVE_HELP="$(vllm serve --help=all 2>/dev/null || vllm serve --help 2>/dev/null || true)" + if grep -q -- '--structured-outputs-config' <<<"$VLLM_SERVE_HELP"; then + EVAL_SERVE_ARGS+=(--structured-outputs-config "{\"backend\":\"$SO_BACKEND\"}") + elif grep -q -- '--guided-decoding-backend' <<<"$VLLM_SERVE_HELP"; then + EVAL_SERVE_ARGS+=(--guided-decoding-backend "$SO_BACKEND") + else + echo "WARN: no structured-outputs backend flag in this image; leaving the default in place" >&2 + fi + else + echo "WARN: llguidance unavailable; leaving the default structured-outputs backend (xgrammar) in place" >&2 + fi + fi + + # 300 SWE-bench Lite instances at the sweep's conc would not finish inside + # SWEBENCH_AGENT_TIMEOUT (6h). Accuracy does not depend on the conc point, + # only wall-clock does, so widen serving concurrency for eval and let the + # harness match it. + EVAL_MAX_NUM_SEQS="${EVAL_MAX_NUM_SEQS:-64}" + export SWEBENCH_AGENT_WORKERS="${SWEBENCH_AGENT_WORKERS:-$EVAL_MAX_NUM_SEQS}" + MAX_NUM_SEQS="$EVAL_MAX_NUM_SEQS" +fi + +# Stand-alone fallback only: the shipped DSpark arm pins GPU_MEM_UTIL=0.95 in +# kimik3_fp4_mi355x_mtp.sh to match the upstream AMD reproducer exactly, and +# that wrapper's value wins over these defaults. +# +# 0.88, not the reference's 0.95, is what the GPU-only bring-up cells actually +# survived on cluster:mi355x-amds. 0.95 asks for 273.59 of 287.98 GiB and +# cleared only 2 of 9 cells; observed free memory at startup varies 208-281 GiB +# across nodes, hours apart on the same node. 0.90 (259.19 GiB) comes up clean +# with a 4,302,357-token pool and then dies mid-prefill -- +# HSA_STATUS_ERROR_OUT_OF_RESOURCES on 4 of 8 ranks at num_computed_tokens +# ~362K with kv_cache_usage only ~21% (run 30521327099) -- because the +# transient chunked-prefill workspace scales with context and needs more than +# the ~22 GiB of unreserved headroom it leaves. 0.88 leaves ~28 GiB and served +# 987 requests at ISL p90 374,550 with a 0/165 error rate (run 30453589555). +# +# c1 gets 0.85: it died at num_computed_tokens 373,248 at both 0.88 (runs +# 30453589555, 30525206671) and 0.90 (run 30521327099), always with +# num_running_reqs=1 -- a single lane replays the deepest trajectories with +# nothing to interleave, so it reaches the fatal band early and reliably. One +# trajectory cannot fill the pool anyway, so the ~530K tokens 0.85 gives up buy +# ~13 GiB/rank of headroom for free. +if [ "${CONC:-}" = "1" ]; then + GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.85}" +else + GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.88}" +fi +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-4096}" + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +# Long-context forward passes (~370K tokens under fp8 KV) can exceed +# vLLM's default 300s worker RPC timeout, killing the engine with +# "RPC call to sample_tokens timed out". Widen it. +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-1200}" + +# Patch aiter's Gluon MLA kernel with a fixed version. The b128 kernel is only +# exercised on the fp8 KV path, so only patch when KV_CACHE_DTYPE is fp8. +if [ "${KV_CACHE_DTYPE:-}" = "fp8" ]; then + MLA_GLUON_DST="/usr/local/lib/python3.12/dist-packages/aiter/ops/triton/gluon/mla_gluon.py" + # Pinned to the exact gist revision that produced the first clean fp8 KV + # run (30442578333: c8, 3600s, 696 total tok/s/GPU, TTFT 2.6s). The + # unpinned .../raw/mla_gluon.py URL silently follows the gist HEAD, so a + # later edit would change the kernel under us with no diff in this repo. + MLA_GLUON_SRC="https://gist.githubusercontent.com/seungrokj/f64cb547829360bfb304f5e794d284ac/raw/4b0088c5fecbeffa6544d2da1006b45380aac896/mla_gluon.py" + if [ -f "$MLA_GLUON_DST" ]; then + echo "Patching $MLA_GLUON_DST from gist..." + curl --silent --fail --location "$MLA_GLUON_SRC" -o "$MLA_GLUON_DST" \ + && echo "Patched mla_gluon.py" \ + || echo "WARN: failed to patch mla_gluon.py; leaving the image version in place" >&2 + else + echo "WARN: $MLA_GLUON_DST not found; skipping mla_gluon.py patch" >&2 + fi +fi + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + --moe-backend auto + --tensor-parallel-size "$TP" + "${EP_ARGS[@]}" + --load-format auto + --gpu-memory-utilization "$GPU_MEM_UTIL" + "${MM_ARGS[@]}" + --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --enable-auto-tool-choice + --tool-call-parser kimi_k3 + --reasoning-parser kimi_k3 + "${MAX_MODEL_LEN_ARGS[@]}" + "${PREFIX_CACHE_ARGS[@]}" + "${KV_CACHE_DTYPE_ARGS[@]}" + "${KV_BLOCK_SIZE_ARGS[@]}" + "${EAGER_ARGS[@]}" + "${SPEC_ARGS[@]}" + "${EVAL_SERVE_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh new file mode 100755 index 0000000000..4dad02c02a --- /dev/null +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +# DSpark variant of kimik3_fp4_mi355x.sh. The MI355X launcher routes +# spec-decoding=mtp rows to this suffix, while the shared base recipe owns the +# model, AgentX replay, and eval plumbing. +# +# Keep this wrapper aligned with the upstream AMD Kimi-K3 DSpark reproducer. +# The first AgentX validation is deliberately GPU-only at c1 so a server or +# kernel failure has nowhere else to hide. +export SPEC_DECODE=true +export KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}" +export GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.95}" +export MAX_NUM_SEQS="${MAX_NUM_SEQS:-16}" +export EVAL_MAX_NUM_SEQS="${EVAL_MAX_NUM_SEQS:-128}" +export MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-4096}" +export LANGUAGE_MODEL_ONLY="${LANGUAGE_MODEL_ONLY:-false}" +export PREFIX_CACHING="${PREFIX_CACHING:-auto}" +export ENFORCE_EAGER="${ENFORCE_EAGER:-false}" + +exec "$(dirname "$0")/kimik3_fp4_mi355x.sh" "$@" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b8f6f4fc31..7b1ad24a09 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -616,6 +616,35 @@ kimik2.5-fp4-mi355x-atom-disagg: additional-settings: - "DECODE_NODES=2" +# Kimi-K3 MXFP4 MI355X AgentX, DSpark only. Kimi-K3 agentic is onboarded +# spec-decode-only from day 0 -- the non-DSpark arm is neither run nor +# published (MODELS.md) -- so this is the single shipped key for this +# model/SKU, and it is the AMD sibling of kimik3-fp4-b300-vllm-agentic-dspark +# in configs/nvidia-master.yaml. +# +# TP=8 only: the MXFP4 checkpoint is 1.561 TB decimal (1.420 TiB, 96 +# safetensors), ~195 GB/GPU of the 288 GB part, so TP=4 cannot load. Upstream +# strategy_min_gpus agrees (single_node_tp and multi_node_tep both 8, DEP 16+), +# which is why there is no DP-attention arm either. +kimik3-fp4-mi355x-vllm-agentic-dspark: + image: vllm/vllm-openai-rocm:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + # No dram-utilization: this arm is GPU-only, so the generator resolves + # TOTAL_CPU_DRAM_GB to 0 either way and the field would be inert. + - search-space: + # Exact upstream AMD DSpark reproducer: GPU-only TP8, seven draft tokens, + # probabilistic draft sampling, TRITON_MLA, and real block rejection. + # Start at c1; expand only after the AgentX-fast preflight is healthy. + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1] } + + dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..fc850013db 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5340,6 +5340,16 @@ - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-dspark + description: + - "Add the Kimi-K3 MXFP4 MI355X AgentX DSpark recipe with Inferact/Kimi-K3-DSpark, seven draft tokens, probabilistic draft sampling, and TRITON_MLA on ROCm. This is the only key for this model/SKU: Kimi-K3 agentic is onboarded spec-decode-only from day 0, so the non-DSpark arm is neither run nor published (MODELS.md), and the key name matches the NVIDIA sibling kimik3-fp4-b300-vllm-agentic-dspark." + - "Align the first AgentX validation with the upstream AMD reproducer: GPU-only TP8 at concurrency 1, real block rejection, --gpu-memory-utilization 0.95, --max-num-batched-tokens 4096, --max-num-seqs 16, multimodal encoder data parallelism, and no --enforce-eager or explicit prefix-cache override." + - "Use VLLM_ROCM_USE_AITER=1, SAFETENSORS_FAST_GPU=1, AITER_SITUV2_A8W4=1, AITER_BF16_FP8_MOE_BOUND=0, and VLLM_USE_BREAKABLE_CUDAGRAPH=0 with the stock vllm/vllm-openai-rocm:kimi-k3 image." + - "The recipe is GPU-only (kv-offloading: none) and rejects a matrix row that asks for a host tier rather than silently ignoring it; the LMCache / SimpleCPUOffload / OffloadingConnector plumbing the bring-up sweeps used is not carried, and returns with the arm that needs it." + - "Passes spec-decoding and the kv-offload-backend metadata through to the agentic-eval job in e2e-tests.yml, which previously hardcoded spec-decoding: none and rendered an object-valued kv-offload-backend as a broken string. Without the spec-decoding passthrough the DSpark eval job would resolve to the non-DSpark script name." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2403 + - config-keys: - kimik2.6-fp4-b200-dynamo-vllm description: diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 75a67c1c58..6e3769080f 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -268,7 +268,36 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" set -x - salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" + # Node denylist. mia1-p01-g14 chronically carries a stale ~32 GiB/GPU + # allocation from previous tenants, so vLLM refuses to start on it: only + # ~256 of 287.98 GiB is free and any gpu-memory-utilization above ~0.888 + # trips "Free memory on device cuda:N ... is less than desired GPU memory + # utilization". It took out kimik3 cells in runs 30330955808 and + # 30331297999, and the same node did the same thing to the kimik2.7 -tune2 + # cells. + # + # A BROAD denylist did not work: after excluding g11/g14/g15/g18, run + # 30412966635 still lost all three cells -- to g19 (hipErrorIllegalAddress), + # g17 (warmup_failure) and g16 (srun node failure), none of them on the + # list. Free memory on these nodes varies hour to hour, so a wide static + # list mostly just shrinks an already-contended pool and lengthens queue + # time. Don't reintroduce one. + # + # g11 is the exception and is excluded by default. It is not a + # free-memory-headroom case; it fails hard and across unrelated configs: + # - kimik3 kvnone c2, run 30425820793: warmup_failure + # - kimik3 vllm-simple-fp8 c8, run 30428781263: forward passes stalled + # (returned=5 for 13 min at zero throughput), then EngineCore died + # - plus the earlier kimik2.7 -tune2 cells noted above + # Four configs, one node, same outcome. Set SALLOC_EXCLUDE_NODES="" to + # clear it, or to a comma-separated list to widen it for a specific run. + SALLOC_EXCLUDE_NODES="${SALLOC_EXCLUDE_NODES-mia1-p01-g11}" + EXCLUDE_ARG="" + if [ -n "$SALLOC_EXCLUDE_NODES" ]; then + EXCLUDE_ARG="--exclude=$SALLOC_EXCLUDE_NODES" + echo "salloc excluding nodes: $SALLOC_EXCLUDE_NODES" + fi + salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" $EXCLUDE_ARG JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" @@ -306,6 +335,14 @@ else export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" fi + # Kimi-K3's MXFP4 checkpoint is 1.561 TB decimal (1.420 TiB, 96 + # safetensors) and does not fit the node-local /var/lib NVMe hub cache + # alongside anything else, so it is pre-staged once on the NFS share like + # MiniMax-M3. + if [[ "$MODEL" == moonshotai/Kimi-K3* ]]; then + export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" + fi + SCRIPT_BASE="${EXP_NAME%%_*}_${PRECISION}_mi355x" SCRIPT_FW="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" SCRIPT_FALLBACK="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh" diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index 0489cb4649..3f00860c45 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -409,6 +409,23 @@ def test_e2e_workflow_cannot_dispatch_database_ingest() -> None: assert "INFX_FRONTEND_PAT" not in workflow +def test_e2e_agentic_eval_preserves_matrix_transport_settings() -> None: + workflow = yaml.load( + (REPO_ROOT / ".github/workflows/e2e-tests.yml").read_text(), + Loader=yaml.BaseLoader, + ) + inputs = workflow["jobs"]["test-sweep-agentic-evals"]["with"] + + assert inputs["kv-offload-backend"] == ( + "${{ matrix.config['kv-offload-backend'].name }}" + ) + assert inputs["kv-offload-backend-metadata"] == ( + "${{ matrix.config['kv-offload-backend']" + " && toJson(matrix.config['kv-offload-backend']) || '' }}" + ) + assert inputs["spec-decoding"] == "${{ matrix.config.spec-decoding }}" + + def test_priority_classifier_runs_for_enabled_actions() -> None: scenario = { **_PR, diff --git a/utils/matrix_logic/test_kimik3_mi355x_dspark.py b/utils/matrix_logic/test_kimik3_mi355x_dspark.py new file mode 100644 index 0000000000..fcd9b8a13c --- /dev/null +++ b/utils/matrix_logic/test_kimik3_mi355x_dspark.py @@ -0,0 +1,70 @@ +from pathlib import Path +import re + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MTP_WRAPPER = ( + REPO_ROOT / "benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh" +) +BASE_RECIPE = REPO_ROOT / "benchmarks/single_node/agentic/kimik3_fp4_mi355x.sh" +MASTER_CONFIG = REPO_ROOT / "configs/amd-master.yaml" + + +def _shell_default(script: str, variable: str) -> str: + match = re.search( + rf'export {re.escape(variable)}="\$\{{{re.escape(variable)}:-(.*?)\}}"', + script, + ) + assert match is not None, f"{variable} must have an overridable default" + return match.group(1) + + +def test_dspark_wrapper_matches_oren_rocm_baseline() -> None: + wrapper = MTP_WRAPPER.read_text() + + assert _shell_default(wrapper, "KV_CACHE_DTYPE") == "auto" + assert _shell_default(wrapper, "GPU_MEM_UTIL") == "0.95" + assert _shell_default(wrapper, "MAX_NUM_SEQS") == "16" + assert _shell_default(wrapper, "EVAL_MAX_NUM_SEQS") == "128" + assert _shell_default(wrapper, "MAX_NUM_BATCHED_TOKENS") == "4096" + assert _shell_default(wrapper, "LANGUAGE_MODEL_ONLY") == "false" + assert _shell_default(wrapper, "PREFIX_CACHING") == "auto" + assert _shell_default(wrapper, "ENFORCE_EAGER") == "false" + + +def test_dspark_defaults_to_real_block_rejection() -> None: + recipe = BASE_RECIPE.read_text() + spec_section = recipe.split("# ---- Eval-only path", maxsplit=1)[0].split( + "SPEC_ARGS=()", maxsplit=1 + )[1] + + assert '\\"rejection_sample_method\\":\\"block\\"' in spec_section + assert "synthetic_acceptance_length" not in spec_section + + +def test_reference_prefix_cache_mode_emits_no_override() -> None: + recipe = BASE_RECIPE.read_text() + prefix_section = recipe.split("# The upstream DSpark config", maxsplit=1)[0].split( + "PREFIX_CACHE_ARGS=", maxsplit=1 + )[1] + + assert "auto)" in prefix_section + assert "PREFIX_CACHE_ARGS=()" in prefix_section + + +def test_dspark_diagnostic_matrix_is_gpu_only_c1() -> None: + master = yaml.safe_load(MASTER_CONFIG.read_text()) + search_space = master["kimik3-fp4-mi355x-vllm-agentic-dspark"]["scenarios"][ + "agentic-coding" + ][0]["search-space"] + + assert search_space == [ + { + "tp": 8, + "kv-offloading": "none", + "spec-decoding": "mtp", + "conc-list": [1], + } + ]