diff --git a/workloads/devstral_small_2_24b_b200.yaml b/workloads/devstral_small_2_24b_b200.yaml new file mode 100644 index 0000000..de7ac8b --- /dev/null +++ b/workloads/devstral_small_2_24b_b200.yaml @@ -0,0 +1,29 @@ +# Devstral-Small-2-24B-Instruct-2512 (FP8) on B200 (1xB200, TP=1) -- accuracy-only (gsm8k) +# Reference gsm8k (vLLM CI gsm8k harness, 5-shot, 1319 questions): 0.834 +# --limit-mm-per-prompt {"image":0} disables the vision path (gsm8k is text-only) +# and avoids the multimodal image-fetch init on this Mistral vision model. +name: devstral_small_2_24b-b200 +gpu: B200 +num_gpus: 1 +nightly: true + +vllm: + model: mistralai/Devstral-Small-2-24B-Instruct-2512 + serve_args: >- + --tensor-parallel-size 1 + --enforce-eager + --max-model-len 4096 + --limit-mm-per-prompt {"image":0} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 4096 + max_gen_toks: 512 diff --git a/workloads/gemma_4_31b_nvfp4_b200.yaml b/workloads/gemma_4_31b_nvfp4_b200.yaml new file mode 100644 index 0000000..488bbfc --- /dev/null +++ b/workloads/gemma_4_31b_nvfp4_b200.yaml @@ -0,0 +1,26 @@ +# Gemma-4-31B-IT NVFP4 on B200 (1xB200, TP=1) -- accuracy-only (gsm8k) +# Reference gsm8k (vLLM CI gsm8k harness, 5-shot, 1319 questions): 0.702 +name: gemma_4_31b_nvfp4-b200 +gpu: B200 +num_gpus: 1 +nightly: true + +vllm: + model: nvidia/Gemma-4-31B-IT-NVFP4 + serve_args: >- + --tensor-parallel-size 1 + --enforce-eager + --max-model-len 4096 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 4096 + max_gen_toks: 512 diff --git a/workloads/llama_3_3_70b_nvfp4_b200.yaml b/workloads/llama_3_3_70b_nvfp4_b200.yaml new file mode 100644 index 0000000..b05b232 --- /dev/null +++ b/workloads/llama_3_3_70b_nvfp4_b200.yaml @@ -0,0 +1,26 @@ +# Llama-3.3-70B-Instruct NVFP4 on B200 (1xB200, TP=1) -- accuracy-only (gsm8k) +# Reference gsm8k (vLLM CI gsm8k harness, 5-shot, 1319 questions): 0.911 +name: llama_3_3_70b_nvfp4-b200 +gpu: B200 +num_gpus: 1 +nightly: true + +vllm: + model: nvidia/Llama-3.3-70B-Instruct-NVFP4 + serve_args: >- + --tensor-parallel-size 1 + --enforce-eager + --max-model-len 4096 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 4096 + max_gen_toks: 512 diff --git a/workloads/mistral_small_4_119b_nvfp4_b200.yaml b/workloads/mistral_small_4_119b_nvfp4_b200.yaml new file mode 100644 index 0000000..fef0c90 --- /dev/null +++ b/workloads/mistral_small_4_119b_nvfp4_b200.yaml @@ -0,0 +1,29 @@ +# Mistral-Small-4-119B-2603 NVFP4 on B200 (1xB200, TP=1) -- accuracy-only (gsm8k) +# Reference gsm8k (vLLM CI gsm8k harness, 5-shot, 1319 questions): 0.905 +# --limit-mm-per-prompt {"image":0} disables the vision path (gsm8k is text-only) +# and avoids the multimodal image-fetch init on this Mistral vision model. +name: mistral_small_4_119b_nvfp4-b200 +gpu: B200 +num_gpus: 1 +nightly: true + +vllm: + model: mistralai/Mistral-Small-4-119B-2603-NVFP4 + serve_args: >- + --tensor-parallel-size 1 + --enforce-eager + --max-model-len 4096 + --limit-mm-per-prompt {"image":0} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 4096 + max_gen_toks: 512