diff --git a/README.md b/README.md index b362b5e42b8b..e1324129fd92 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ TensorRT LLM [![python](https://img.shields.io/badge/python-3.10-green)](https://www.python.org/downloads/release/python-31012/) [![cuda](https://img.shields.io/badge/cuda-13.2.1-green)](https://developer.nvidia.com/cuda-downloads) [![torch](https://img.shields.io/badge/torch-2.11.0-green)](https://pytorch.org) -[![version](https://img.shields.io/badge/release-1.3.0rc22-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) +[![version](https://img.shields.io/badge/release-1.3.0rc23-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) [![license](https://img.shields.io/badge/license-Apache%202-blue)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) [Architecture](https://nvidia.github.io/TensorRT-LLM/developer-guide/overview.html)   |   [Performance](https://nvidia.github.io/TensorRT-LLM/developer-guide/perf-overview.html)   |   [Examples](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html)   |   [Documentation](https://nvidia.github.io/TensorRT-LLM/)   |   [Roadmap](https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap) diff --git a/docs/source/_static/config_db.json b/docs/source/_static/config_db.json index 7671934a6911..8af081380d9c 100644 --- a/docs/source/_static/config_db.json +++ b/docs/source/_static/config_db.json @@ -179,6 +179,30 @@ "model_display_name": "MiniMax-M3 (MXFP8)", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", "scenario": "Max Throughput" + }, + { + "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_filename": "deepseek-v4-pro-latency.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_path": "examples/configs/curated/deepseek-v4-pro-latency.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", + "gpu_compatibility": "B200", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", + "model_url": "", + "scenario": "Min Latency" + }, + { + "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_filename": "deepseek-v4-pro-throughput.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_path": "examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", + "gpu_compatibility": "B200", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", + "model_url": "", + "scenario": "Max Throughput" } ], "entries": [ @@ -1321,6 +1345,82 @@ "osl": 1024, "performance_profile": "Max Throughput" }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "concurrency": 4, + "config_filename": "1k1k_tp4_conc4.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Min Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "concurrency": 8, + "config_filename": "1k1k_tp4_conc8.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "concurrency": 16, + "config_filename": "1k1k_tp4_conc16.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "concurrency": 256, + "config_filename": "1k1k_tp4_conc256.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 1024, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Max Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml", "concurrency": 2048, @@ -1338,6 +1438,82 @@ "osl": 8192, "performance_profile": "High Throughput" }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "concurrency": 4, + "config_filename": "8k1k_tp4_conc4.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Min Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "concurrency": 8, + "config_filename": "8k1k_tp4_conc8.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "concurrency": 16, + "config_filename": "8k1k_tp4_conc16.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, + { + "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "concurrency": 256, + "config_filename": "8k1k_tp4_conc256.yaml", + "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", + "gpu": "B200_NVL", + "gpu_display": "4xB200_NVL", + "isl": 8192, + "model": "nvidia/DeepSeek-R1-0528-FP4-v2", + "model_display_name": "DeepSeek-R1 (NVFP4)", + "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", + "num_gpus": 4, + "osl": 1024, + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" + }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml", "concurrency": 1024, @@ -1353,7 +1529,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 4, "osl": 1024, - "performance_profile": "Min Latency" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml", @@ -1421,7 +1597,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency" + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml", @@ -1472,7 +1650,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced" + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml", @@ -1489,7 +1669,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced" + "performance_profile": "Balanced", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml", @@ -1506,7 +1688,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml", @@ -1523,7 +1707,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml", @@ -1812,7 +1998,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency" + "performance_profile": "Low Latency", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml", @@ -1880,7 +2068,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml", @@ -1897,7 +2087,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml", @@ -1914,7 +2106,9 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "High Throughput", + "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", + "validated_trtllm_version": "1.3.0rc14" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml", @@ -3532,6 +3726,10 @@ "display_name": "DeepSeek-R1", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, + "deepseek-ai/DeepSeek-V4-Pro": { + "display_name": "deepseek-ai/DeepSeek-V4-Pro", + "url": "" + }, "nvidia/DeepSeek-R1-0528-FP4-v2": { "display_name": "DeepSeek-R1 (NVFP4)", "url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2" diff --git a/docs/source/_static/config_selector.css b/docs/source/_static/config_selector.css index ca84bb991f2a..43c5a4aa7bc6 100644 --- a/docs/source/_static/config_selector.css +++ b/docs/source/_static/config_selector.css @@ -1,3 +1,6 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +/* SPDX-License-Identifier: Apache-2.0 */ + .trtllm-config-selector { border: 1px solid var(--pst-color-shadow); border-radius: 14px; @@ -29,6 +32,10 @@ align-items: start; } +.trtllm-config-selector__field[hidden] { + display: none; +} + .trtllm-config-selector__label { display: flex; align-items: center; diff --git a/docs/source/_static/config_selector.js b/docs/source/_static/config_selector.js index e8ef62550a6b..ca053655923a 100644 --- a/docs/source/_static/config_selector.js +++ b/docs/source/_static/config_selector.js @@ -1,14 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + (function () { "use strict"; let dbPromise = null; let widgetId = 0; - const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency"]; + const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency", "profile"]; const GROUP_LABELS = { model: "Model", topology: "GPU(s)", islOsl: "ISL / OSL", concurrency: "Concurrency", + profile: "Profile", }; function $(root, sel) { @@ -58,6 +62,7 @@ state.concurrency != null && state.concurrency !== "" ? String(state.concurrency) : "", + profile: state.profile || "", }; } @@ -138,9 +143,22 @@ function profileLabel(profile) { const text = String(profile || "").trim(); + const labels = { + latency: "Latency", + balanced: "Balanced", + throughput: "Throughput", + }; + if (labels[text]) return labels[text]; return text || "Unknown Profile"; } + function profileOption(profile) { + return { + value: profile, + label: profileLabel(profile), + }; + } + function formatProfileSummary(profiles) { const labels = uniqBy(profiles.map((profile) => profileLabel(profile)), (label) => label); if (!labels.length) { @@ -194,7 +212,12 @@ .map((group) => concurrencyOption(group)) .sort((a, b) => sortNums(a.concurrency, b.concurrency)); - return { model, topology, islOsl, concurrency }; + const profile = uniqBy( + entries.filter((entry) => entry.profile).map((entry) => profileOption(entry.profile)), + (option) => option.value + ); + + return { model, topology, islOsl, concurrency, profile }; } function filterEntriesByState(entries, state) { @@ -214,6 +237,9 @@ if (normalizedState.concurrency) { if (String(entry.concurrency) !== normalizedState.concurrency) return false; } + if (normalizedState.profile && entry.profile !== normalizedState.profile) { + return false; + } return true; }); } @@ -362,7 +388,8 @@ normalizedState.model && normalizedState.topology && normalizedState.islOsl && - normalizedState.concurrency + normalizedState.concurrency && + (!groups.profile.options.length || normalizedState.profile) ) { return "Selection did not resolve to a single configuration."; } @@ -389,6 +416,12 @@ }; } + function validatedCommitUrl(commit) { + const normalized = String(commit || "").trim().toLowerCase(); + if (!/^[0-9a-f]{40}$/.test(normalized)) return ""; + return `https://github.com/NVIDIA/TensorRT-LLM/commit/${normalized}`; + } + function isFileProtocol() { return window.location.protocol === "file:"; } @@ -608,14 +641,14 @@ : allCurated ).map(normalizeEntry); - // curatedIndex lives outside normalizeState's scope — it is preserved - // across Object.assign(state, view.state) because normalizeState only - // touches the four filter keys. + // curatedIndex lives outside normalizeState's scope and is preserved + // across Object.assign(state, view.state). const state = { model: "", topology: "", islOsl: "", concurrency: "", + profile: "", curatedIndex: null, }; @@ -685,6 +718,7 @@ const selTopo = mkOptionGroup("GPU(s)", 2); const selSeq = mkOptionGroup("ISL / OSL", 3); const selConc = mkSelectField("Concurrency", `trtllm-conc-${id}`, 4); + const selProfile = mkSelectField("Profile", `trtllm-profile-${id}`, 5); form.appendChild(selModel.wrap); @@ -707,6 +741,7 @@ form.appendChild(selTopo.wrap); form.appendChild(selSeq.wrap); form.appendChild(selConc.wrap); + form.appendChild(selProfile.wrap); const output = el("div", { class: "trtllm-config-selector__output" }); const cmdPre = el("pre", { class: "trtllm-config-selector__cmd" }, [ @@ -835,7 +870,10 @@ return { type: "model-select" }; } if (activeEl === selConc.select) { - return { type: "select" }; + return { type: "select", key: "concurrency" }; + } + if (activeEl === selProfile.select) { + return { type: "select", key: "profile" }; } if ( activeEl.classList && @@ -857,7 +895,8 @@ return; } if (descriptor.type === "select") { - selConc.select.focus(); + if (descriptor.key === "profile") selProfile.select.focus(); + else selConc.select.focus(); return; } if (descriptor.type !== "button") return; @@ -997,8 +1036,8 @@ selectEl.dataset.status = (selectedOption && selectedOption.status) || "idle"; } - function setSelectOptions(selectEl, group) { - const previousValue = state.concurrency || ""; + function setSelectOptions(selectEl, group, stateKey, placeholder) { + const previousValue = state[stateKey] || ""; const visibleOptions = group.options.filter( (option) => option.status !== "incompatible" ); @@ -1006,7 +1045,7 @@ selectEl.appendChild( el("option", { value: "", - text: visibleOptions.length ? "Select concurrency" : "No concurrency available", + text: visibleOptions.length ? `Select ${placeholder}` : `No ${placeholder} available`, }) ); for (const option of visibleOptions) { @@ -1077,7 +1116,12 @@ setOptionButtons(selTopo.options, "topology", view.groups.topology); setOptionButtons(selSeq.options, "islOsl", view.groups.islOsl); - setSelectOptions(selConc.select, view.groups.concurrency); + setSelectOptions(selConc.select, view.groups.concurrency, "concurrency", "concurrency"); + const hasProfileChoices = + Boolean(view.state.concurrency) && + view.groups.profile.options.some((option) => option.status !== "incompatible"); + selProfile.wrap.hidden = !hasProfileChoices; + setSelectOptions(selProfile.select, view.groups.profile, "profile", "profile"); const code = cmdPre.querySelector("code"); if (curatedSelected) { @@ -1132,6 +1176,26 @@ } else { meta.appendChild(el("span", { text: e.config_path || "" })); } + if (e.validated_trtllm_version) { + meta.appendChild( + el("span", { + text: ` \u00b7 Validated with TensorRT-LLM ${e.validated_trtllm_version}`, + }) + ); + } + const commitUrl = validatedCommitUrl(e.validated_trtllm_commit); + if (commitUrl) { + meta.appendChild(el("span", { text: " \u00b7 Commit: " })); + meta.appendChild( + el("a", { + class: "trtllm-config-selector__configLink", + href: commitUrl, + target: "_blank", + rel: "noopener", + text: e.validated_trtllm_commit.slice(0, 12), + }) + ); + } currentEntry = e; resetYamlPanel(); @@ -1161,6 +1225,12 @@ selConc.select.addEventListener("change", () => { state.concurrency = selConc.select.value; + state.profile = ""; + render(); + }); + + selProfile.select.addEventListener("change", () => { + state.profile = selProfile.select.value; render(); }); @@ -1218,6 +1288,7 @@ formatCuratedCommand: formatCommand, nextStateAfterSelection, normalizeState, + validatedCommitUrl, }; } diff --git a/examples/configs/database/database.py b/examples/configs/database/database.py index 3dcdd13c0300..af33a4dab41b 100644 --- a/examples/configs/database/database.py +++ b/examples/configs/database/database.py @@ -13,13 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. - import logging +import re +from collections import defaultdict from pathlib import Path -from typing import Any, Dict, Iterator, List, Tuple +from typing import Any, Iterator, Literal import yaml -from pydantic import BaseModel, Field, RootModel, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + RootModel, + field_validator, + model_validator, +) logger = logging.getLogger(__name__) @@ -30,11 +39,22 @@ LOW_LATENCY_CONCURRENCY_THRESHOLD = 8 HIGH_THROUGHPUT_CONCURRENCY_THRESHOLD = 32 KEY_PROFILES = {"Min Latency", "Balanced", "Max Throughput"} +PROFILE_DISPLAY_NAMES = { + "latency": "Min Latency", + "balanced": "Balanced", + "throughput": "Max Throughput", +} +PROFILE_ORDER = {profile: idx for idx, profile in enumerate(PROFILE_DISPLAY_NAMES)} +VALIDATED_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") +VALIDATED_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._+-]*$") +Profile = Literal["latency", "balanced", "throughput"] class CuratedRecipe(BaseModel): """A curated (hand-tuned) recipe entry.""" + model_config = ConfigDict(extra="forbid") + model: str = Field(description="HuggingFace model ID") arch: str = Field(description="Model architecture class name") config_path: str = Field(description="Relative path to YAML config") @@ -51,7 +71,7 @@ def _validate_config_path(cls, v: str) -> str: return v -class CuratedRecipeList(RootModel[List[CuratedRecipe]]): +class CuratedRecipeList(RootModel[list[CuratedRecipe]]): """Validated list of curated recipe entries.""" @classmethod @@ -75,15 +95,58 @@ def __len__(self) -> int: class Recipe(BaseModel): """Recipe record for scenario list.""" - model: str = Field(description="Model name") - gpu: str = Field(description="GPU name") - isl: int = Field(description="Input sequence length") - osl: int = Field(description="Output sequence length") - concurrency: int = Field(description="Concurrency") - config_path: str = Field(description="Configuration path") - num_gpus: int = Field(description="Number of GPUs") - - def load_config(self) -> Dict[str, Any]: + model_config = ConfigDict(extra="forbid") + + model: str = Field(min_length=1, description="Model name") + arch: str = Field(min_length=1, description="Model architecture class name") + gpu: str = Field(min_length=1, description="GPU name") + isl: PositiveInt = Field(description="Input sequence length") + osl: PositiveInt = Field(description="Output sequence length") + concurrency: PositiveInt = Field(description="Concurrency") + config_path: str = Field(min_length=1, description="Configuration path") + num_gpus: PositiveInt = Field(description="Number of GPUs") + profile: Profile | None = Field( + default=None, + description="Profile discriminator used only when the workload key has multiple configs", + ) + validated_trtllm_commit: str | None = Field( + default=None, + description="Full TensorRT-LLM commit SHA against which the recipe was validated", + ) + validated_trtllm_version: str | None = Field( + default=None, + description="TensorRT-LLM release version reported by the validation source", + ) + + @field_validator("validated_trtllm_commit") + @classmethod + def _validate_commit(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip().lower() + if not VALIDATED_COMMIT_PATTERN.fullmatch(normalized): + raise ValueError("validated_trtllm_commit must be a full 40-character Git SHA") + return normalized + + @field_validator("validated_trtllm_version") + @classmethod + def _validate_version(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + if not VALIDATED_VERSION_PATTERN.fullmatch(normalized): + raise ValueError("validated_trtllm_version must be a release-tag-safe version") + return normalized + + @model_validator(mode="after") + def _validate_provenance_pair(self) -> "Recipe": + if bool(self.validated_trtllm_commit) != bool(self.validated_trtllm_version): + raise ValueError( + "validated_trtllm_commit and validated_trtllm_version must be provided together" + ) + return self + + def load_config(self) -> dict[str, Any]: """Load and return the YAML config at config_path.""" config_relative_path = Path(self.config_path) # Ensure config path is within the repo root @@ -96,7 +159,41 @@ def load_config(self) -> Dict[str, Any]: return yaml.safe_load(f) -class RecipeList(RootModel[List[Recipe]]): +class RecipeList(RootModel[list[Recipe]]): + @model_validator(mode="after") + def _validate_conflict_profiles(self) -> "RecipeList": + groups = defaultdict(list) + for recipe in self.root: + key = ( + recipe.model, + recipe.gpu, + recipe.num_gpus, + recipe.isl, + recipe.osl, + recipe.concurrency, + ) + groups[key].append(recipe) + + required_profiles = {"latency", "throughput"} + for key, recipes in groups.items(): + profiles = [recipe.profile for recipe in recipes] + if len(recipes) == 1: + if profiles[0] is not None: + raise ValueError(f"profile is only allowed for conflicting workload key {key}") + continue + profile_set = set(profiles) + if ( + None in profile_set + or len(profile_set) != len(profiles) + or not required_profiles.issubset(profile_set) + ): + raise ValueError( + "conflicting workload key " + f"{key} must have exactly one latency and throughput profile, " + "with an optional balanced profile" + ) + return self + @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": """Load and validate recipe list from YAML file.""" @@ -132,12 +229,15 @@ def assign_profile(num_recipes: int, idx: int, concurrency: int) -> str: return "High Throughput" -def select_key_recipes(recipes: List[Recipe]) -> List[Tuple[Recipe, str]]: +def select_key_recipes(recipes: list[Recipe]) -> list[tuple[Recipe, str]]: """Select key recipes (min latency, balanced, max throughput) from a list of recipes.""" if not recipes: return [] - sorted_recipes = sorted(recipes, key=lambda r: r.concurrency) + sorted_recipes = sorted( + recipes, + key=lambda r: (r.concurrency, PROFILE_ORDER.get(r.profile, -1)), + ) n = len(sorted_recipes) result = [] diff --git a/examples/configs/database/lookup.yaml b/examples/configs/database/lookup.yaml index 523cda14c412..d773fa361596 100644 --- a/examples/configs/database/lookup.yaml +++ b/examples/configs/database/lookup.yaml @@ -558,6 +558,8 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -582,6 +584,8 @@ concurrency: 32 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -590,6 +594,8 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -598,6 +604,8 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -606,6 +614,8 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -750,6 +760,8 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -782,6 +794,8 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -790,6 +804,8 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -798,6 +814,8 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -1566,3 +1584,83 @@ concurrency: 1536 config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc1536.yaml num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + arch: DeepseekV3ForCausalLM + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml + num_gpus: 4 + validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 + validated_trtllm_version: 1.3.0rc14 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..c9986a111b5b --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml @@ -0,0 +1,14 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_seq_len: 8192 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..bb43a3699209 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml @@ -0,0 +1,24 @@ +max_batch_size: 64 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +speculative_config: + decoding_type: MTP + max_draft_len: 1 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +max_num_tokens: 1216 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..ade4e5b1de21 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml @@ -0,0 +1,19 @@ +max_batch_size: 4 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..7f49059c11b1 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml @@ -0,0 +1,19 @@ +max_batch_size: 8 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index ea4b03e17984..3db82c4881df 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -1,24 +1,19 @@ -max_batch_size: 512 +max_batch_size: 128 cuda_graph_config: enable_padding: true - max_batch_size: 32 -enable_attention_dp: true + max_batch_size: 128 print_iter_log: true kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.8 stream_interval: 10 moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 + backend: TRTLLM speculative_config: decoding_type: MTP - max_draft_len: 1 + max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 3072 -max_seq_len: 2068 +max_num_tokens: 1600 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 9031655d28fd..9bc761a79868 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -1,7 +1,6 @@ -max_batch_size: 512 cuda_graph_config: enable_padding: true - max_batch_size: 256 + max_batch_size: 64 enable_attention_dp: true print_iter_log: true kv_cache_config: @@ -14,11 +13,7 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 -speculative_config: - decoding_type: MTP - max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 2112 -max_seq_len: 2068 +max_seq_len: 8192 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index 5d7f74990a3a..03c9e6b27901 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 32 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -15,5 +15,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 3136 -max_seq_len: 2068 +max_num_tokens: 1216 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index 68eab71800d9..cf0bb6a44862 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 4 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 3136 -max_seq_len: 2068 +max_num_tokens: 1152 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index c239238e8f0d..3348357986a6 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,7 @@ +max_batch_size: 64 cuda_graph_config: enable_padding: true - max_batch_size: 16 -enable_attention_dp: true + max_batch_size: 64 print_iter_log: true kv_cache_config: dtype: fp8 @@ -9,14 +9,11 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 speculative_config: decoding_type: MTP - max_draft_len: 1 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_seq_len: 2068 +max_num_tokens: 1344 +max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..283c8ccd18a3 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml @@ -0,0 +1,19 @@ +max_batch_size: 16 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..aa75a4c8d4c5 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml @@ -0,0 +1,24 @@ +max_batch_size: 64 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +speculative_config: + decoding_type: MTP + max_draft_len: 1 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +max_num_tokens: 8384 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..594aae2eaba4 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml @@ -0,0 +1,19 @@ +max_batch_size: 4 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..2dbc0435be33 --- /dev/null +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml @@ -0,0 +1,19 @@ +max_batch_size: 8 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 +stream_interval: 10 +moe_config: + backend: TRTLLM +speculative_config: + decoding_type: MTP + max_draft_len: 3 +tensor_parallel_size: 4 +moe_expert_parallel_size: 1 +trust_remote_code: true +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index baaa644ca539..143fe4bba30b 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 32 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index 93df61d6fa56..251ed48facac 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 64 cuda_graph_config: enable_padding: true max_batch_size: 64 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8384 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index 6a6b9fb25c63..97ce51217d82 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 512 +max_batch_size: 4 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 8 +moe_expert_parallel_size: 1 trust_remote_code: true -max_num_tokens: 10304 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index b39f07478407..6729a6fa5c6d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -max_batch_size: 256 +max_batch_size: 16 cuda_graph_config: enable_padding: true max_batch_size: 16 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8768 -max_seq_len: 9416 +max_num_tokens: 8320 +max_seq_len: 9472 diff --git a/examples/constraints.txt b/examples/constraints.txt index 641bb9c89346..01ee827c13ea 100644 --- a/examples/constraints.txt +++ b/examples/constraints.txt @@ -1,3 +1,3 @@ -tensorrt_llm==1.3.0rc22 +tensorrt_llm==1.3.0rc23 evaluate~=0.4.1 rouge_score~=0.1.2 diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5524a74ef603..c647275e6de6 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -236,6 +236,53 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2 } } +// Scrape the SLURM job output log for a device / driver / interconnect fault +// signature and return the matched signature itself, or "" for no match. +// +// Device faults (CUDA/NVLink/ECC/driver) print into job-output.log but never +// reach the stage exception chain -- the tracker squashes a failed job to +// `exit 1` -- so classify() otherwise sees only a generic failure and cannot +// steer the retry off the bad node. This is a GATE only: the returned signature +// is folded into a fresh exception so FailureClassifier.PATTERN_CATALOG (the +// authoritative list) makes the real retry/severity decision. A signature the +// catalog does not recognize simply falls through to a normal rethrow. +// App-induced CUDA errors (illegal memory access, unspecified launch failure, +// OOM) are deliberately excluded -- the OpenSearch stage data shows those are +// overwhelmingly code regressions, not node faults, and must not trigger a +// node-avoiding retry. +// +// grep -o returns only the matched signature (not the whole line), so a long +// log line cannot truncate the signature out of the result before it reaches +// classify(). Each alternative must therefore be catalog-exact: it must match +// (via `.` wildcards for shell-hostile chars) the full catalog substring, so +// grep -o emits text that still contains the catalog pattern. +def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath) { + def deviceFaultRegex = "cudaErrorMapBufferObjectFailed|mapping of buffer object failed|" + + "uncorrectable NVLink error|cudaErrorNvlinkUncorrectable|CUDA_ERROR_SYSTEM_NOT_READY|" + + "uncorrectable ECC error|CUDA_ERROR_ECC_UNCORRECTABLE|has fallen off the bus|GPU is lost|" + + "Unable to determine the device handle for GPU|RmInitAdapter failed|Failed to initialize NVML|" + + "could... communicate with the NVIDIA driver|CUDA_ERROR_DEVICE_UNAVAILABLE|" + + "no CUDA-capable device is detected|CUDA_ERROR_UNKNOWN: 999|CUDA unknown error|" + + "CUDA-capable device.s. is/are busy or unavailable" + try { + // Wrap the body in `bash -c` so it is shell-agnostic: cluster login shells + // are often csh/tcsh, which can't parse this bash test/pipe/redirection + // syntax. The login shell only has to run `bash -c ''`. + return Utils.exec( + pipeline, + script: Utils.sshUserCmd(remote, + "\"bash -c 'if [ -f \\\"${remoteLogPath}\\\" ]; then grep -aioE \\\"${deviceFaultRegex}\\\" \\\"${remoteLogPath}\\\" 2>/dev/null | tail -n 1 | cut -c1-500; fi'\""), + returnStdout: true, + numRetries: 1, + )?.trim() + } catch (InterruptedException e) { + throw e + } catch (Exception scrapeEx) { + pipeline.echo("Ignorable warning: could not scrape ${remoteLogPath} for device faults on ${remote.host}: ${scrapeEx.message}") + return "" + } +} + // `postTag` uniquifies the uploaded tar filename, the Artifactory guard key and // the locally-staged result XMLs when the same stageName is uploaded more than // once in a build (e.g. SLURM infra-failure retries). First attempt passes "". @@ -1889,6 +1936,23 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "Original failure: ${e.message}", e) } + // A terminal FAILED state may be a node/device fault whose + // signature (CUDA/NVLink/ECC/driver) printed only into the SLURM + // job output log, never into this exception chain. Scrape the log + // and, on a hit, surface the matched line into a fresh exception + // so the authoritative catalog (FailureClassifier.classify at the + // runLLMTestlistWithSbatch caller) can match it and steer the retry + // off the bad node. A miss falls through to the plain rethrow. + if (slurmState == "FAILED") { + def deviceHit = scrapeSlurmLogForDeviceFault(pipeline, remote, slurmJobLogPath) + if (deviceHit) { + echo "[INFRA-RETRY] ${stageName}: device-fault signature in SLURM job ${slurmJobId} log; " + + "surfacing to classifier: ${deviceHit}" + throw new Exception( + "Device/interconnect fault on SLURM node during job ${slurmJobId} for ${stageName}: " + + "${deviceHit} | original: ${e.message}") + } + } echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobId} terminal state=${slurmState ?: 'unknown'}; " + "deferring to failure classifier." throw e diff --git a/scripts/generate_config_database_tests.py b/scripts/generate_config_database_tests.py index 10fdfbc66e26..74c7050cd89f 100644 --- a/scripts/generate_config_database_tests.py +++ b/scripts/generate_config_database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,12 +71,17 @@ def generate_server_name(recipe: Recipe) -> str: """Generate a unique server name from recipe.""" model_slug = recipe.model.replace("/", "_").replace("-", "_").replace(".", "_") name = f"{model_slug}_{recipe.isl}_{recipe.osl}_conc{recipe.concurrency}_gpu{recipe.num_gpus}" + if recipe.profile: + name = f"{name}_{recipe.profile}" return name def generate_client_name(recipe: Recipe) -> str: """Generate client config name.""" - return f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" + name = f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" + if recipe.profile: + name = f"{name}_{recipe.profile}" + return name def recipe_to_server_config(recipe: Recipe, llm_api_config: dict) -> dict: diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 70fa8145c55c..fc73c29d8d90 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -32,6 +32,8 @@ from examples.configs.database.database import ( # noqa: E402 CURATED_LIST_PATH, DATABASE_LIST_PATH, + PROFILE_DISPLAY_NAMES, + PROFILE_ORDER, CuratedRecipeList, RecipeList, assign_profile, @@ -90,9 +92,9 @@ "display_name": "Kimi-K2-Thinking (NVFP4)", "url": "https://huggingface.co/nvidia/Kimi-K2-Thinking-NVFP4", }, - "MiniMaxAI/MiniMax-M3": { - "display_name": "MiniMax-M3 (BF16)", - "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3", + "MiniMaxAI/MiniMax-M3-MXFP8": { + "display_name": "MiniMax-M3 (MXFP8)", + "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", }, } @@ -114,6 +116,9 @@ class RecipeRow: config_filename: str config_github_url: str config_raw_url: str + profile: str | None + validated_trtllm_commit: str | None + validated_trtllm_version: str | None @dataclass(frozen=True) @@ -167,7 +172,7 @@ def build_curated_rows(yaml_path: Path) -> list[CuratedRow]: return rows -def build_rows(yaml_path) -> list[RecipeRow]: +def build_rows(yaml_path: Path) -> list[RecipeRow]: recipe_list = RecipeList.from_yaml(Path(yaml_path)) model_groups = defaultdict(lambda: defaultdict(list)) @@ -189,7 +194,7 @@ def build_rows(yaml_path) -> list[RecipeRow]: for key in sorted_keys: entries = subgroups[key] - entries.sort(key=lambda x: x.concurrency) + entries.sort(key=lambda x: (x.concurrency, PROFILE_ORDER.get(x.profile, -1))) for idx, entry in enumerate(entries): gpu = entry.gpu @@ -200,7 +205,11 @@ def build_rows(yaml_path) -> list[RecipeRow]: conc = entry.concurrency config_path = entry.config_path - profile = assign_profile(len(entries), idx, conc) + performance_profile = ( + PROFILE_DISPLAY_NAMES[entry.profile] + if entry.profile + else assign_profile(len(entries), idx, conc) + ) command = f"trtllm-serve {model} --config ${{TRTLLM_DIR}}/{config_path}" @@ -224,18 +233,23 @@ def build_rows(yaml_path) -> list[RecipeRow]: concurrency=conc, config_path=config_path, gpu_display=gpu_display, - performance_profile=profile, + performance_profile=performance_profile, command=command, config_filename=config_filename, config_github_url=config_github_url, config_raw_url=config_raw_url, + profile=entry.profile, + validated_trtllm_commit=entry.validated_trtllm_commit, + validated_trtllm_version=entry.validated_trtllm_version, ) ) return rows -def generate_json(yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None): +def generate_json( + yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None +) -> None: rows = build_rows(yaml_path) source_path = Path(yaml_path) @@ -266,7 +280,9 @@ def generate_json(yaml_path: Path, output_file: Path, curated_yaml_path: Path | payload = { "source": source, "models": models, - "entries": [asdict(r) for r in rows], + "entries": [ + {key: value for key, value in asdict(row).items() if value is not None} for row in rows + ], "curated_entries": curated_entries, } diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py index f9f21a911a95..e819f1690728 100644 --- a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -16,10 +16,13 @@ (the FlashInfer prefill kernel does NOT apply L2 norm internally; the ``use_qk_l2norm_in_kernel`` parameter on ``flashinfer.chunk_gated_delta_rule`` is currently a dead arg, see ``flashinfer/gdn_prefill.py:317-356``). - * Pre-gather and post-scatter of indexed SSM state (FlashInfer requires - packed ``[num_seqs, H, V, K]`` fp32 initial/output state). TRT-LLM's GDN - state pool uses the same ``[N, H, V, K]`` logical layout, so the adapter - casts/gathers/scatters without transposing the last two dims. + * Pre-gather and post-scatter of indexed SSM state into FlashInfer's packed + ``[num_seqs, H, V, K]`` layout. TRT-LLM's GDN state pool uses the same + ``[N, H, V, K]`` logical layout, so the adapter gathers/scatters without + transposing the last two dims. The SM100/SM103 kernel carries the recurrent + state in fp32 in TMEM regardless of the initial/output-state I/O dtype, so + the round-trip stays in the native pool dtype (bf16/fp16) there with no + precision change; only SM90/SM120 need an fp32 up-cast/down-cast. This module is only imported when ``TLLM_USE_FLASHINFER_GDN_PREFILL=1`` is set at process start; do not import it lazily inside hot paths. @@ -34,6 +37,7 @@ gather_cast_vk_to_fp32_vk, ) from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd +from tensorrt_llm._utils import is_sm_100f # Mirror the @torch.compiler.disable on the legacy Triton wrapper @@ -103,10 +107,18 @@ def chunk_gated_delta_rule( q3 = l2norm_fwd(q3) k3 = l2norm_fwd(k3) - # --- Step 4: gather initial state and cast dtype --------------------- + # --- Step 4: gather initial state (+ cast dtype only when required) --- # TRT-LLM's GDN kernels and FlashInfer both use [N, H, V, K] state layout. - # Fuse gather + cast-to-fp32 + contiguous into a single Triton kernel. - gathered_init = gather_cast_vk_to_fp32_vk(initial_state, initial_state_indices) + # The SM100/SM103 kernel carries the recurrent state in fp32 in TMEM + # regardless of the initial/output-state I/O dtype (the state tensors are + # only the gmem load/store format), so passing bf16/fp16 state there is + # numerically identical to the fp32 round-trip while moving half the bytes. + # SM90/SM120 still require fp32 state. Fuse gather (+ optional cast) and + # contiguous into a single Triton kernel. + state_dtype = initial_state.dtype if is_sm_100f() else torch.float32 + gathered_init = gather_cast_vk_to_fp32_vk( + initial_state, initial_state_indices, out_dtype=state_dtype + ) # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its @@ -126,7 +138,10 @@ def chunk_gated_delta_rule( ) if need_state: num_seqs = cu_seqlens.shape[0] - 1 - state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=torch.float32) + # Match the initial-state dtype (native bf16/fp16 on SM100/SM103, else + # fp32); FlashInfer writes the final state in this dtype and the scatter + # below adapts to the destination pool dtype without an extra cast. + state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=state_dtype) out_packed, out_state = flashinfer.chunk_gated_delta_rule( q=q3, k=k3, @@ -158,8 +173,9 @@ def chunk_gated_delta_rule( ) out_state = None - # --- Step 7: cast state back, scatter / return --------------------- - # Fuse cast (fp32 -> initial_state.dtype) + optional indexed scatter into a + # --- Step 7: cast state back (if needed), scatter / return --------- + # Fuse cast (out_state.dtype -> destination dtype; a no-op on SM100/SM103 + # where both are the native pool dtype) + optional indexed scatter into a # single Triton pass, mirroring Step 4. The inplace branch writes only the # slots named by ``initial_state_indices`` and leaves the rest untouched. if inplace_indexed_state_update: diff --git a/tensorrt_llm/_torch/modules/fla/fused_state_io.py b/tensorrt_llm/_torch/modules/fla/fused_state_io.py index c5885f5b6011..ee420993fec3 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_state_io.py +++ b/tensorrt_llm/_torch/modules/fla/fused_state_io.py @@ -79,8 +79,16 @@ def _gather_cast_vk_to_fp32_vk_kernel( def gather_cast_vk_to_fp32_vk( initial_state: torch.Tensor, initial_state_indices: Optional[torch.Tensor], + out_dtype: Optional[torch.dtype] = None, ) -> torch.Tensor: - """Fused ``initial_state[indices].to(fp32).contiguous()`` for ``[N, H, V, K]`` state.""" + """Fused ``initial_state[indices].to(out_dtype).contiguous()`` for ``[N, H, V, K]`` state. + + ``out_dtype`` defaults to ``torch.float32``, the dtype the SM90/SM120 + FlashInfer GDN prefill kernels require. On the SM100/SM103 kernel, which + reads native bf16/fp16 state and casts to fp32 internally, pass + ``initial_state.dtype`` to gather without an up-cast (paired with a matching + scatter that skips the down-cast). + """ assert initial_state.dim() == 4, f"initial_state must be 4D, got {initial_state.shape}" n_pool, h, v, k = initial_state.shape if initial_state_indices is not None: @@ -93,7 +101,9 @@ def gather_cast_vk_to_fp32_vk( # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the full K and V dimensions. # entire (K, V) plane per (seq, head). Larger tiles save grid overhead; # smaller tiles improve occupancy at small num_seqs * H. - output = torch.empty(num_seqs, h, v, k, dtype=torch.float32, device=initial_state.device) + if out_dtype is None: + out_dtype = torch.float32 + output = torch.empty(num_seqs, h, v, k, dtype=out_dtype, device=initial_state.device) block_v = min(v, 128) block_k = min(k, 128) num_v_blocks = triton.cdiv(v, block_v) diff --git a/tensorrt_llm/version.py b/tensorrt_llm/version.py index 49aa6938a3f6..f4cbad2b6663 100644 --- a/tensorrt_llm/version.py +++ b/tensorrt_llm/version.py @@ -12,4 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.3.0rc22" +__version__ = "1.3.0rc23" diff --git a/tests/unittest/executor/test_proxy_fast_death.py b/tests/unittest/executor/test_proxy_fast_death.py index 0032bc393cd2..c72ed8c21b1a 100644 --- a/tests/unittest/executor/test_proxy_fast_death.py +++ b/tests/unittest/executor/test_proxy_fast_death.py @@ -53,6 +53,7 @@ def _bare_proxy(): proxy._results = {} # Set so the __del__ -> shutdown() path is a clean no-op at GC time. proxy.workers_started = False + proxy._multi_frontend_ipc_dir = None return proxy diff --git a/tests/unittest/tools/test_config_database_sync.py b/tests/unittest/tools/test_config_database_sync.py index 83e5670f6783..2a20d2707052 100644 --- a/tests/unittest/tools/test_config_database_sync.py +++ b/tests/unittest/tools/test_config_database_sync.py @@ -22,6 +22,7 @@ from pathlib import Path import pytest +import yaml REPO_ROOT = Path(__file__).parent.parent.parent.parent.resolve() EXPECTED_MODEL_METADATA = { @@ -58,6 +59,7 @@ sys.modules[_db_spec.name] = _db_module _db_spec.loader.exec_module(_db_module) generate_tests = _db_module.generate_tests +generate_server_name = _db_module.generate_server_name TEST_LIST_PATH = _db_module.TEST_LIST_PATH PERF_SANITY_DIR = _db_module.PERF_SANITY_DIR @@ -85,6 +87,9 @@ def test_config_db_json_generation(self): int(r.osl), int(r.concurrency), r.config_path, + r.profile, + r.validated_trtllm_commit, + r.validated_trtllm_version, ) for r in recipes } @@ -126,6 +131,9 @@ def test_config_db_json_generation(self): int(e.get("osl")), int(e.get("concurrency")), e.get("config_path"), + e.get("profile"), + e.get("validated_trtllm_commit"), + e.get("validated_trtllm_version"), ) self.assertIn( key, @@ -152,6 +160,104 @@ def test_config_db_json_generation(self): "Generated config_db.json is missing entries from lookup.yaml.", ) + def test_profile_and_validated_commit_metadata(self) -> None: + commit = "93CB6518B6D6DBD6095748189E626DB731F44545" + version = "1.3.0rc14" + recipes = [] + for profile in ("latency", "balanced", "throughput"): + recipes.append( + { + "model": "example/model", + "arch": "ExampleForCausalLM", + "gpu": "B200_NVL", + "num_gpus": 8, + "isl": 1024, + "osl": 1024, + "concurrency": 256, + "config_path": f"examples/configs/database/example_{profile}.yaml", + "profile": profile, + "validated_trtllm_commit": commit, + "validated_trtllm_version": version, + } + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + yaml_path = Path(tmp_dir) / "lookup.yaml" + output_path = Path(tmp_dir) / "config_db.json" + yaml_path.write_text(yaml.safe_dump(recipes), encoding="utf-8") + generate_json(yaml_path, output_path) + payload = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual( + [entry["profile"] for entry in payload["entries"]], + ["latency", "balanced", "throughput"], + ) + self.assertEqual( + {entry["performance_profile"] for entry in payload["entries"]}, + {"Min Latency", "Balanced", "Max Throughput"}, + ) + self.assertTrue( + all(entry["validated_trtllm_commit"] == commit.lower() for entry in payload["entries"]) + ) + self.assertTrue( + all(entry["validated_trtllm_version"] == version for entry in payload["entries"]) + ) + + parsed = RecipeList.model_validate(recipes) + server_names = [generate_server_name(recipe) for recipe in parsed] + self.assertEqual(len(server_names), len(set(server_names))) + self.assertTrue(server_names[0].endswith("_latency")) + + def test_recipe_metadata_validation(self) -> None: + base = { + "model": "example/model", + "arch": "ExampleForCausalLM", + "gpu": "B200_NVL", + "num_gpus": 8, + "isl": 1024, + "osl": 1024, + "concurrency": 256, + "config_path": "examples/configs/database/example.yaml", + } + + with self.assertRaisesRegex(ValueError, "full 40-character Git SHA"): + RecipeList.model_validate( + [ + { + **base, + "validated_trtllm_commit": "deadbeef", + "validated_trtllm_version": "1.3.0rc14", + } + ] + ) + + with self.assertRaisesRegex(ValueError, "must be provided together"): + RecipeList.model_validate([{**base, "validated_trtllm_commit": "a" * 40}]) + + with self.assertRaisesRegex(ValueError, "profile is only allowed"): + RecipeList.model_validate([{**base, "profile": "balanced"}]) + + endpoint_conflict = [ + {**base, "config_path": "latency.yaml", "profile": "latency"}, + {**base, "config_path": "throughput.yaml", "profile": "throughput"}, + ] + parsed = RecipeList.model_validate(endpoint_conflict) + self.assertEqual([recipe.profile for recipe in parsed], ["latency", "throughput"]) + + missing_endpoint = [ + {**base, "config_path": "latency.yaml", "profile": "latency"}, + {**base, "config_path": "balanced.yaml", "profile": "balanced"}, + ] + with self.assertRaisesRegex(ValueError, "exactly one latency and throughput"): + RecipeList.model_validate(missing_endpoint) + + duplicate_profile = [ + {**base, "config_path": "latency-a.yaml", "profile": "latency"}, + {**base, "config_path": "latency-b.yaml", "profile": "latency"}, + ] + with self.assertRaisesRegex(ValueError, "exactly one latency and throughput"): + RecipeList.model_validate(duplicate_profile) + @pytest.mark.skip(reason="https://nvbugs/6337224") def test_config_database_tests_sync(self): """Test that config database test files are synchronized with lookup.yaml. diff --git a/tests/unittest/tools/test_config_selector.js b/tests/unittest/tools/test_config_selector.js index d5f57bb7cb6b..ce489982cc69 100644 --- a/tests/unittest/tools/test_config_selector.js +++ b/tests/unittest/tools/test_config_selector.js @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + const assert = require("node:assert/strict"); const fs = require("node:fs"); const path = require("node:path"); @@ -77,6 +80,50 @@ test("selector exports a pure view-model helper for compatibility logic", () => assert.equal(typeof selector.createSelectorViewModel, "function"); }); +test("selector builds a validated TensorRT-LLM commit link", () => { + const selector = loadSelectorExports(); + const commit = "93cb6518b6d6dbd6095748189e626db731f44545"; + assert.equal( + selector.validatedCommitUrl(commit), + `https://github.com/NVIDIA/TensorRT-LLM/commit/${commit}` + ); + assert.equal(selector.validatedCommitUrl("not-a-commit"), ""); +}); + +test("selector resolves same-concurrency conflicts by explicit profile", () => { + const selector = loadSelectorExports(); + const entries = ["latency", "balanced", "throughput"].map((profile) => ({ + model: "example/model", + gpu: "B200_NVL", + gpu_display: "8xB200_NVL", + num_gpus: 8, + isl: 1024, + osl: 1024, + concurrency: 256, + profile, + performance_profile: profile, + config_path: `${profile}.yaml`, + command: profile, + })); + const baseState = { + model: "example/model", + topology: "8|B200_NVL", + islOsl: "1024|1024", + concurrency: "256", + }; + + const unresolved = selector.createSelectorViewModel(entries, {}, baseState); + assert.equal(unresolved.finalEntries.length, 3); + assert.match(unresolved.message, /Available Profile options/); + + const resolved = selector.createSelectorViewModel(entries, {}, { + ...baseState, + profile: "balanced", + }); + assert.equal(resolved.finalEntries.length, 1); + assert.equal(resolved.resolvedEntry.profile, "balanced"); +}); + test("selector keeps both 4x and 8x B200 topologies visible for DeepSeek NVFP4", () => { const { selector, payload, entries } = setupDeepSeekNvfp4(); @@ -91,7 +138,7 @@ test("selector keeps both 4x and 8x B200 topologies visible for DeepSeek NVFP4", assert.ok(findOption(view.groups.topology, "8|B200_NVL")); }); -test("selector marks 1024/1024 unavailable for 4x B200 on DeepSeek NVFP4", () => { +test("selector includes automated 1024/1024 configs for 4x B200", () => { const { selector, payload, entries } = setupDeepSeekNvfp4(); const view = selector.createSelectorViewModel(entries, payload.models, { @@ -101,16 +148,31 @@ test("selector marks 1024/1024 unavailable for 4x B200 on DeepSeek NVFP4", () => concurrency: "", }); - assert.equal(findOption(view.groups.islOsl, "1024|1024").status, "incompatible"); + assert.equal(findOption(view.groups.islOsl, "1024|1024").status, "available"); assert.equal(findOption(view.groups.islOsl, "1024|8192").status, "available"); assert.equal(findOption(view.groups.islOsl, "8192|1024").status, "available"); }); test("selector preserves invalid active selections and explains the clash", () => { - const { selector, payload, entries } = setupDeepSeekNvfp4(); - - const view = selector.createSelectorViewModel(entries, payload.models, { - model: DEEPSEEK_NVFP4_MODEL, + const selector = loadSelectorExports(); + const entries = [ + [4, 1024, 8192], + [4, 8192, 1024], + [8, 1024, 1024], + ].map(([numGpus, isl, osl]) => ({ + model: "example/model", + gpu: "B200_NVL", + gpu_display: `${numGpus}xB200_NVL`, + num_gpus: numGpus, + isl, + osl, + concurrency: 1, + config_path: `${numGpus}-${isl}-${osl}.yaml`, + command: `${numGpus}-${isl}-${osl}`, + })); + + const view = selector.createSelectorViewModel(entries, {}, { + model: "example/model", topology: "4|B200_NVL", islOsl: "1024|1024", concurrency: "",