diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 10411ad069..8090747d46 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -271,7 +271,10 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - timeout-minutes: 480 + # Full-context AgentX warmup can legitimately exceed the fixed-sequence + # eight-hour envelope at high concurrency. Keep one hour beyond the + # recipe's 12-hour Slurm limit for checkout, staging, and artifact upload. + timeout-minutes: ${{ inputs.scenario-type == 'agentic-coding' && 780 || 480 }} name: >- p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} ${{ inputs.disagg == 'true' && format('{0}P ', inputs.prefill-num-worker) || '' }}(TP${{ inputs.prefill-tp }}${{ inputs.prefill-pp != '' && inputs.prefill-pp != '1' && format('/PP{0}', inputs.prefill-pp) || '' }}${{ inputs.prefill-dcp-size != '' && inputs.prefill-dcp-size != '1' && format('/DCP{0}', inputs.prefill-dcp-size) || '' }}${{ inputs.prefill-pcp-size != '' && inputs.prefill-pcp-size != '1' && format('/PCP{0}', inputs.prefill-pcp-size) || '' }}${{ inputs.prefill-ep != '' && inputs.prefill-ep != '1' && format('/EP{0}', inputs.prefill-ep) || '' }}${{ inputs.prefill-dp-attn == 'true' && '/DPA' || '' }}) @@ -286,7 +289,7 @@ jobs: run: &slurm-cleanup | if command -v squeue >/dev/null 2>&1; then echo "[Slurm] Cleaning up jobs named: ${{ runner.name }} ..." - scancel --name="${{ runner.name }}" || true + scancel --user="$USER" --name="${{ runner.name }}" || true while [ -n "$(squeue --user="$USER" --name='${{ runner.name }}' --noheader --format='%i')" ]; do squeue --user="$USER" --name="${{ runner.name }}" sleep 5 diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index dfb3f7aafc..6082a9a7de 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1577,6 +1577,7 @@ AIPERF_CLI="${AIPERF_VENV}/bin/aiperf" AIPERF_HF_CLI="${AIPERF_VENV}/bin/hf" AIPERF_DEPS_READY=0 AIPERF_FAILED_REQUEST_THRESHOLD="${AIPERF_FAILED_REQUEST_THRESHOLD:-0.10}" +AIPERF_LIVE_FAILED_REQUEST_THRESHOLD="${AIPERF_LIVE_FAILED_REQUEST_THRESHOLD:-$AIPERF_FAILED_REQUEST_THRESHOLD}" agentic_pip_install() { local pip_install=(python3 -m pip install) @@ -1743,6 +1744,7 @@ build_replay_cmd() { local result_dir="$1" local duration="$DURATION" local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}" + local trace_idle_gap_cap_seconds="${AIPERF_TRACE_IDLE_GAP_CAP_SECONDS:-}" # Fast mode minimizes setup by advancing each trajectory lane only once # and shortens profiling to 20 minutes. @@ -1775,11 +1777,23 @@ build_replay_cmd() { REPLAY_CMD+=" --benchmark-duration $duration" REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" - # Fail runs once more than 10% of requests error. This keeps known - # transient low-rate failures from killing long sweeps while still - # catching malformed payloads or server crashes before they get aggregated - # as benchmarkable data. - REPLAY_CMD+=" --failed-request-threshold $AIPERF_FAILED_REQUEST_THRESHOLD" + # Some long AgentX traces contain recorded request-start gaps that would + # otherwise hold a trajectory lane idle for much longer than the useful + # cache-TTL window. When a recipe opts in, cap those gaps independently + # within each root trace during dataset reconstruction. This changes only + # replay timing; it is not a runtime request timeout. + if [ -n "$trace_idle_gap_cap_seconds" ]; then + if ! [[ "$trace_idle_gap_cap_seconds" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "ERROR: AIPERF_TRACE_IDLE_GAP_CAP_SECONDS must be a non-negative number" >&2 + return 1 + fi + REPLAY_CMD+=" --trace-idle-gap-cap-seconds $trace_idle_gap_cap_seconds" + fi + # Fail runs early once the live error ratio crosses the configured limit. + # Recipes with correlated low-concurrency trajectories may allow a larger + # live sample while retaining AIPERF_FAILED_REQUEST_THRESHOLD as the strict + # post-run validity gate below. + REPLAY_CMD+=" --failed-request-threshold $AIPERF_LIVE_FAILED_REQUEST_THRESHOLD" # Sample each trajectory's warmup start position uniformly from # [25%, 75%] of the trace's turn count, clamped by AIPerf to leave at # least one profile turn after warmup. diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh new file mode 100644 index 0000000000..a42c6aba77 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The Kimi K3 DSpark checkpoint publishes its parallel-drafting token as +# `mask_token_id`. Dynamo's serialized draft config reaches vLLM without the +# K3 config-class alias, while vLLM's parallel drafter accepts `pard_token`. +# Build a thin local view of the upstream checkpoint and add that equivalent +# metadata alias without changing vLLM or the checkpoint weights. +python3 - <<'PY' +import json +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +repo_id = "Inferact/Kimi-K3-DSpark" +target = Path("/tmp/Kimi-K3-DSpark") +snapshot = Path(snapshot_download(repo_id=repo_id)) +target.mkdir(parents=True, exist_ok=True) + +for source in snapshot.iterdir(): + if source.name == "config.json": + continue + destination = target / source.name + if destination.is_symlink(): + if destination.resolve() == source.resolve(): + continue + destination.unlink() + elif destination.exists(): + raise RuntimeError(f"Refusing to replace non-symlink path: {destination}") + destination.symlink_to(source) + +config = json.loads((snapshot / "config.json").read_text()) +mask_token_id = config.get("mask_token_id") +if not isinstance(mask_token_id, int): + raise RuntimeError(f"{repo_id} config is missing integer mask_token_id") + +pard_token = config.get("pard_token") +if pard_token not in (None, mask_token_id): + raise RuntimeError( + f"{repo_id} pard_token={pard_token} disagrees with mask_token_id={mask_token_id}" + ) +config["pard_token"] = mask_token_id + +temporary = target / "config.json.tmp" +temporary.write_text(json.dumps(config, indent=2) + "\n") +os.replace(temporary, target / "config.json") +print(f"Prepared {repo_id} compatibility view at {target}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml new file mode 100644 index 0000000000..e7de6389a1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml @@ -0,0 +1,145 @@ +name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" + +# Day-0 GB200 translation of the official throughput-oriented multi_node_dep +# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "12:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. + # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little + # runtime workspace for FlashInfer's MXFP4 MoE kernel. + max-num-seqs: 64 + max-num-batched-tokens: 16384 + speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml new file mode 100644 index 0000000000..703262fc3b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml @@ -0,0 +1,148 @@ +name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" + +# High-concurrency host-DRAM KV-offload variant of the official throughput- +# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU +# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "12:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTHONHASHSEED: "42" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest offload point is c384 / DP4 = 96 sequences per engine. + # Capture even sequence counts: all configured DP4 steady-state batch + # sizes are exact hits, while odd loads pad by at most one sequence. + max-num-seqs: 96 + max-num-batched-tokens: 16384 + speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml new file mode 100644 index 0000000000..bf135c0e33 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml @@ -0,0 +1,137 @@ +name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" + +# Day-0 GB200 translation of the official balanced multi_node_tep profile. +# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes +# with the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + pipeline-parallel-size: 1 + enable-expert-parallel: true + trust-remote-code: true + load-format: "fastsafetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml new file mode 100644 index 0000000000..baed3f19e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml @@ -0,0 +1,135 @@ +name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" + +# Day-0 GB200 translation of the official latency-oriented multi_node_tp +# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + pipeline-parallel-size: 1 + trust-remote-code: true + load-format: "fastsafetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + gpu-memory-utilization: 0.92 + max-num-seqs: 8 + max-num-batched-tokens: 8192 + speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..e8efcddc1a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8835,6 +8835,101 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 8 ep: 8 dp-attn: true + +# Day-0 Kimi K3 AgentX coverage on GB200. The three aggregate srt-slurm +# profiles mirror the serving strategies synthesized by the official recipe: +# latency: multi-node TP16 (4 GB200 nodes / 16 GPUs) +# balanced: multi-node TEP16 (4 GB200 nodes / 16 GPUs) +# throughput: multi-node TP4 x DP4 (4 GB200 nodes / EP16) +# All profiles enable Kimi K3 DSpark level 2 at the committed golden AL 2.51. +# `spec-decoding: mtp` is the existing matrix label used for speculative +# decoding methods; the checked-in recipes select DSpark explicitly. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200 +kimik3-fp4-gb200-dynamo-vllm-agentic: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.61 + search-space: + # Latency oriented: multi_node_tp strategy, TP16 across four GB200 + # nodes. + # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp + - spec-decoding: mtp + conc-list: [1, 2, 4, 8] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 1 + dp-attn: false + # Balanced: multi_node_tep strategy, TEP16 across four GB200 nodes. + # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep + - spec-decoding: mtp + conc-list: [8, 16, 24, 32] + prefill: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 16 + dp-attn: false + # Throughput oriented: official multi_node_dep strategy, DEP16 across + # four GB200 nodes (TP4 x DP4 = EP16, one local DP rank per node). + # The recipe allows up to 3600s for full-context saturation warmup to drain. + # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep + - spec-decoding: mtp + conc-list: [32, 64, 96, 128, 192, 256] + prefill: + num-worker: 1 + tp: 4 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 16 + dp-attn: true + # High-concurrency DEP16 with vLLM Simple CPU KV offloading. c384 + # exercises 384 of the 393 AgentX trajectories and remains below the + # aggregate max-num-seqs capacity of 512 (128 per DP rank). + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "13c59a3" } + conc-list: [128, 192, 256, 384] + prefill: + num-worker: 1 + tp: 4 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 16 + dp-attn: true + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/configs/runners.yaml b/configs/runners.yaml index 5e992eb4e5..79dfcc3757 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -251,6 +251,7 @@ labels: - gb200-nv_0 - gb200-nv_1 - gb200-nv_2 + - gb200-nv_3 cluster:gb300-nv: - gb300-nv_0 - gb300-nv_1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..3b1d7ebda4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,17 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic + description: + - "Add Kimi K3 MXFP4 GB200 AgentX performance points following the official vLLM recipe: TP16 (conc 1-8), TEP16 (conc 8-32), and TP4 x DP4 = EP16 (conc 32-256). https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200" + - "Test vLLM Simple CPU KV offloading on the Kimi K3 GB200 DEP16 throughput profile at AgentX concurrency 128, 192, 256, and 384." + - "Correct the pure TP16 result identity to EP1 so it remains distinct from the TEP16 point at concurrency 8." + - "Use the upstream vllm/vllm-openai:kimi-k3 image with the pinned Kimi K3-enabled Dynamo 1.3.0 source build installed in-container, and enable Kimi K3 DSpark level 2 with the prescribed 2.51 golden acceptance length." + - "Use FlashInfer TRT-LLM MoE for DSpark DEP to avoid DeepGEMM grid-barrier failures under MRV2." + - "Load DEP weights with standard safetensors to avoid fastsafetensors transient HBM exhaustion with FlashInfer MoE." + - "Size DEP CUDA-graph captures to the actual per-engine frontier concurrency (64 regular, 96 offload), using even-sequence captures for the high-concurrency offload arm to preserve the full 1M context while leaving FlashInfer runtime workspace." + - "Pin the Qwen B200/B300 AgentX AIPerf warmup-handoff revision, cap recorded per-trace idle gaps at 300 seconds, report vLLM cached prompt tokens, and let low-concurrency TP collect a larger live error sample before applying the unchanged 10% final validity gate." + - "Allow high-concurrency full-context AgentX jobs to complete the canonical warmup and one-hour profile with a 12-hour Slurm limit and 13-hour workflow envelope." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2404 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 7bccc43e9c..376217f2b2 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -156,6 +156,12 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then if [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" + elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + # Load Kimi K3 from node-local NVMe for faster startup. The checkpoint + # must be pre-staged at this exact path on every allocated GB200 node. + # This alias matches model.path in the checked-in AgentX recipes. + export MODEL_PATH="/mnt/numa1/models/Kimi-K3" + export SRT_SLURM_MODEL_PREFIX="kimi-k3" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then # FP4 checkpoint on compute-visible Lustre (the /mnt/numa1 path is gone # on watchtower compute nodes). Use the base DeepSeek-V4-Pro checkpoint, @@ -178,7 +184,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" + echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" exit 1 fi else @@ -189,7 +195,7 @@ NGINX_IMAGE="nginx:1.27.4" uses_watchtower_shared_fs() { case "$MODEL_PREFIX" in - minimaxm2.5|minimaxm3|kimik2.5|qwen3.5) return 0 ;; + minimaxm2.5|minimaxm3|kimik2.5|kimik3|qwen3.5) return 0 ;; esac # dsv4 multinode runs only under dynamo-vllm on watchtower, which likewise # needs the srt-slurm workspace/outputs on a compute-visible shared FS @@ -362,8 +368,7 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses the same pinned cquil11/srt-slurm-nv commit as - # launch_gb300-nv.sh — everything the agentic recipes need is there: + # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) @@ -373,10 +378,21 @@ if [[ "$IS_AGENTIC" == "1" ]]; then # must reach the agentic_srt.sh srun) git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout de59739b172e507e15ebf145bfe305f606e82fbf - mkdir -p recipes/vllm/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + if [[ "$MODEL_PREFIX" == "kimik3" ]]; then + # Kimi K3 additionally needs vLLM TP groups within data parallel and + # every DP engine metrics endpoint exposed to AIPerf. + git checkout b1fb626fbdbfe3306dcb51cb181ab35861ec3b1c + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh" \ + configs/kimik3-dspark-config-compat.sh + else + git checkout de59739b172e507e15ebf145bfe305f606e82fbf + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -534,25 +550,28 @@ cat srtslurm.yaml echo "Running make setup..." make setup ARCH=aarch64 || exit 1 -# Export eval-related env vars for srt-slurm post-benchmark eval +# Export eval-related env vars for srt-slurm post-benchmark eval. Current +# Watchtower runners keep GITHUB_WORKSPACE on Lustre, so compute nodes can +# mount it directly; avoid copying the checkout from Lustre back to Lustre. +# Retain staging as a fallback for runners whose workspace is node-local. export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" -# Watchtower: pyxis mounts INFMAX_WORKSPACE into the container, but -# GITHUB_WORKSPACE is under /home/slurm-shared/ which compute nodes -# can't see. Stage the relevant subset to shared FS and repoint -# INFMAX_WORKSPACE there. rsync excludes the srt-slurm clone (already -# on shared FS) and .git (not needed in container) for speed. if uses_watchtower_shared_fs; then - SHARED_INFMAX_WORKSPACE="${SHARED_BASE}/infmax-workspace-${RUN_KEY}" - mkdir -p "$SHARED_INFMAX_WORKSPACE" || exit 1 - rsync -a --delete \ - --exclude='.git/' \ - --exclude='srt-slurm*/' \ - --exclude='outputs/' \ - --exclude='LOGS/' \ - --exclude='*.sqsh' \ - "${GITHUB_WORKSPACE}/" "${SHARED_INFMAX_WORKSPACE}/" || exit 1 - export INFMAX_WORKSPACE="$SHARED_INFMAX_WORKSPACE" - echo "Using shared-FS INFMAX_WORKSPACE=$INFMAX_WORKSPACE (compute-visible)" + WORKSPACE_FS_TYPE=$(findmnt -n -o FSTYPE -T "$GITHUB_WORKSPACE" 2>/dev/null || true) + if [[ "$WORKSPACE_FS_TYPE" == "lustre" ]]; then + echo "Using existing Lustre-backed INFMAX_WORKSPACE=$INFMAX_WORKSPACE" + else + SHARED_INFMAX_WORKSPACE="${SHARED_BASE}/infmax-workspace-${RUN_KEY}" + mkdir -p "$SHARED_INFMAX_WORKSPACE" || exit 1 + rsync -a --delete \ + --exclude='.git/' \ + --exclude='srt-slurm*/' \ + --exclude='outputs/' \ + --exclude='LOGS/' \ + --exclude='*.sqsh' \ + "${GITHUB_WORKSPACE}/" "${SHARED_INFMAX_WORKSPACE}/" || exit 1 + export INFMAX_WORKSPACE="$SHARED_INFMAX_WORKSPACE" + echo "Staged node-local workspace to INFMAX_WORKSPACE=$INFMAX_WORKSPACE" + fi fi echo "Submitting job with srtctl..." @@ -565,8 +584,17 @@ if [[ ! -f "$CONFIG_PATH" ]]; then exit 1 fi -# Keep the Slurm job name aligned with the GitHub runner name. -sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" +# Namespace InferenceX allocations so other repositories using the same +# physical runner names cannot cancel them with `scancel --name=gb200-nv_*`. +# Clean up any stale allocation from this InferenceX runner before submitting. +SRT_SLURM_JOB_NAME="inferencex-${RUNNER_NAME}" +if command -v squeue >/dev/null 2>&1; then + scancel --user="$USER" --name="$SRT_SLURM_JOB_NAME" 2>/dev/null || true + while [[ -n "$(squeue --user="$USER" --name="$SRT_SLURM_JOB_NAME" --noheader --format='%i')" ]]; do + sleep 5 + done +fi +sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH" # Optionally inject synthetic acceptance into the recipe's speculative-config # when SYNTHETIC_ACCEPTANCE=true (no-op otherwise). Must run after the name @@ -603,7 +631,10 @@ if [[ "$FRAMEWORK" == "dynamo-sglang" ]]; then elif [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +# srtctl gives the GitHub-provided RUNNER_NAME precedence over config.name. +# Override it only for submission so the rendered #SBATCH job name retains +# the InferenceX namespace used above. +SRTCTL_OUTPUT=$(RUNNER_NAME="$SRT_SLURM_JOB_NAME" srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) echo "$SRTCTL_OUTPUT" JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+') @@ -617,6 +648,18 @@ fi echo "Extracted JOB_ID: $JOB_ID" +# The workflow-level cleanup keys off the physical runner name, while this +# launcher uses a repository-specific Slurm name to avoid cross-repo +# collisions. Always clean up the exact submitted allocation on exit. +cleanup_srt_job() { + local rc=$? + scancel "$JOB_ID" 2>/dev/null || true + return "$rc" +} +trap cleanup_srt_job EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP + # Use the JOB_ID to find the logs directory # srtctl creates logs in outputs/JOB_ID/logs/ LOGS_DIR="outputs/$JOB_ID/logs" @@ -642,43 +685,54 @@ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then exit 1 fi - # Find all result subdirectories - RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) - - if [ -z "$RESULT_SUBDIRS" ]; then - echo "Warning: No result subdirectories found in $LOGS_DIR" + if [[ "$IS_AGENTIC" == "1" ]]; then + # The custom benchmark runs inside the compute-visible + # INFMAX_WORKSPACE mount. Its aggregation step writes one + # ${RESULT_FILENAME}_conc.json there per point; stage those files + # back to GITHUB_WORKSPACE for the workflow guard and artifact upload. + copy_agentic_results \ + "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" \ + "$RESULT_FILENAME" || exit 1 else - # Process results from all configurations - for result_subdir in $RESULT_SUBDIRS; do - echo "Processing result subdirectory: $result_subdir" - - # Extract configuration info from directory name - CONFIG_NAME=$(basename "$result_subdir") - - # Find all result JSON files - RESULT_FILES=$(find "$result_subdir" -name "results_concurrency_*.json" 2>/dev/null) - - for result_file in $RESULT_FILES; do - if [ -f "$result_file" ]; then - # Extract metadata from filename - # Files may be "results_concurrency_N_gpus_G_ctx_C_gen_D.json" (disagg) or "results_concurrency_N_gpus_G.json" (non-disagg) - filename=$(basename "$result_file") - concurrency=$(echo "$filename" | sed -n 's/results_concurrency_\([0-9]*\)_gpus_.*/\1/p') - gpus=$(echo "$filename" | sed -n 's/results_concurrency_[0-9]*_gpus_\([0-9][0-9]*\).*/\1/p') - ctx=$(echo "$filename" | sed -n 's/.*_ctx_\([0-9]*\)_gen_.*/\1/p') - gen=$(echo "$filename" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p') - - echo "Processing concurrency $concurrency with $gpus GPUs (ctx: $ctx, gen: $gen): $result_file" - - if [ -n "$ctx" ] && [ -n "$gen" ]; then - WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}_ctx_${ctx}_gen_${gen}.json" - else - WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}.json" + # Find all fixed-sequence result subdirectories. + RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) + + if [ -z "$RESULT_SUBDIRS" ]; then + echo "Warning: No result subdirectories found in $LOGS_DIR" + else + # Process results from all configurations + for result_subdir in $RESULT_SUBDIRS; do + echo "Processing result subdirectory: $result_subdir" + + # Extract configuration info from directory name + CONFIG_NAME=$(basename "$result_subdir") + + # Find all result JSON files + RESULT_FILES=$(find "$result_subdir" -name "results_concurrency_*.json" 2>/dev/null) + + for result_file in $RESULT_FILES; do + if [ -f "$result_file" ]; then + # Extract metadata from filename + # Files may be "results_concurrency_N_gpus_G_ctx_C_gen_D.json" (disagg) or "results_concurrency_N_gpus_G.json" (non-disagg) + filename=$(basename "$result_file") + concurrency=$(echo "$filename" | sed -n 's/results_concurrency_\([0-9]*\)_gpus_.*/\1/p') + gpus=$(echo "$filename" | sed -n 's/results_concurrency_[0-9]*_gpus_\([0-9][0-9]*\).*/\1/p') + ctx=$(echo "$filename" | sed -n 's/.*_ctx_\([0-9]*\)_gen_.*/\1/p') + gen=$(echo "$filename" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p') + + echo "Processing concurrency $concurrency with $gpus GPUs (ctx: $ctx, gen: $gen): $result_file" + + if [ -n "$ctx" ] && [ -n "$gen" ]; then + WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}_ctx_${ctx}_gen_${gen}.json" + else + WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}.json" + fi + copy_to_workspace "$result_file" "$WORKSPACE_RESULT_FILE" || exit 1 fi - copy_to_workspace "$result_file" "$WORKSPACE_RESULT_FILE" || exit 1 - fi + done done - done + fi fi echo "All result files processed" diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index c26b4f1690..770694e9bd 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -34,6 +34,15 @@ copy_to_workspace() { local source_file="$1" local destination_file="$2" + # A compute-visible runner workspace may be mounted directly into the + # benchmark container. In that case the staged result already is the + # workflow artifact, so copying it onto itself would fail with cp's + # "same file" error even though the benchmark succeeded. + if [[ -e "$destination_file" && "$source_file" -ef "$destination_file" ]]; then + echo "Result already present at $destination_file" + return 0 + fi + if ! cp "$source_file" "$destination_file"; then echo "ERROR: failed to copy $source_file to $destination_file" >&2 return 1 @@ -41,6 +50,36 @@ copy_to_workspace() { echo "Copied $(basename "$source_file") to $destination_file" } +copy_agentic_results() { + local source_dir="$1" + local workspace="$2" + local result_filename="$3" + local result_file + local copied=0 + + if [[ ! -d "$source_dir" ]]; then + echo "ERROR: agentic result directory not found at $source_dir" >&2 + return 1 + fi + + while IFS= read -r -d '' result_file; do + copy_to_workspace \ + "$result_file" \ + "$workspace/$(basename "$result_file")" || return 1 + copied=$((copied + 1)) + done < <( + find "$source_dir" -maxdepth 1 -type f \ + -name "${result_filename}_conc*.json" -print0 + ) + + if [[ "$copied" -eq 0 ]]; then + echo "ERROR: no ${result_filename}_conc*.json results found in $source_dir" >&2 + return 1 + fi + + echo "Copied $copied agentic result file(s)" +} + copy_eval_artifacts() { local eval_dir="$1" local workspace="$2" diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py new file mode 100644 index 0000000000..c027cc9b2f --- /dev/null +++ b/runners/test_slurm_utils.py @@ -0,0 +1,57 @@ +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" + + +def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "-c", command, "bash", *(str(arg) for arg in args)], + check=False, + capture_output=True, + text=True, + ) + + +def test_copy_agentic_results_stages_only_matching_points(tmp_path: Path) -> None: + source = tmp_path / "source" + workspace = tmp_path / "workspace" + source.mkdir() + workspace.mkdir() + (source / "run_conc1.json").write_text('{"conc": 1}\n') + (source / "run_conc16.json").write_text('{"conc": 16}\n') + (source / "other_conc1.json").write_text('{"conc": 1}\n') + + result = run_bash( + 'source "$1"; copy_agentic_results "$2" "$3" run', + SLURM_UTILS, + source, + workspace, + ) + + assert result.returncode == 0, result.stderr + assert sorted(path.name for path in workspace.iterdir()) == [ + "run_conc1.json", + "run_conc16.json", + ] + + +def test_copy_agentic_results_fails_when_aggregate_is_missing( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + workspace = tmp_path / "workspace" + source.mkdir() + workspace.mkdir() + + result = run_bash( + 'source "$1"; copy_agentic_results "$2" "$3" run', + SLURM_UTILS, + source, + workspace, + ) + + assert result.returncode != 0 + assert "no run_conc*.json results found" in result.stderr diff --git a/utils/aiperf b/utils/aiperf index be758d6218..ed057829b7 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit be758d6218268171e2957fbec9d4f557275bca2d +Subproject commit ed057829b78d25d79ce6f3b87763d48fe50363f5