[Model] Support Qwen3.8 Series

README.md

@@ -76,7 +76,7 @@ Note that some VLMs may not be able to run under certain transformer versions, w
 - **Please use** `transformers==4.48.0` (or `4.46.0`) **for**: `LLaVA-Next series` (e.g., `llava-hf/llava-v1.6-vicuna-7b-hf`).
 - **Please use** `transformers==latest` **for**: `PaliGemma-3B`, `Chameleon series`, `Video-LLaVA-7B-HF`, `Ovis series`, `Mantis series`, `MiniCPM-V2.6`, `OmChat-v2.0-13B-sinlge-beta`, `Idefics-3`, `GLM-4v-9B`, `VideoChat2-HD`, `RBDash_72b`, `Llama-3.2 series`, `Kosmos series`.
 - **Please use** `transformers==4.50.3` (or `4.46.1` or `4.51` or `4.53`) **for**: `Molmo series`.
-- **Please use** `transformers>=5.2.0` **for**: `Qwen3.5 series`.
+- **Please use** `transformers>=5.2.0` **for**: `Qwen3.5 series`, `Qwen3.8 series`.
 
 **Torchvision Version Recommendation:**
 

tests/test_all_qwen3_8_models.py

@@ -0,0 +1,121 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+from vlmeval.config import qwen3_8_series, supported_VLM
+
+
+class TestAllQwen3_8Models(unittest.TestCase):
+
+    def test_all_15_models_configuration(self):
+        """Verify each of the 15 Qwen 3.8 models has correct configuration in supported_VLM."""
+        self.assertEqual(len(qwen3_8_series), 15)
+        for name, partial_func in qwen3_8_series.items():
+            self.assertIn(name, supported_VLM, f"{name} not found in supported_VLM")
+            func = partial_func.func
+            keywords = partial_func.keywords
+            print(f"Verified config for {name} -> {func.__name__} (keys: {list(keywords.keys())})")
+
+    @patch('transformers.AutoModelForImageTextToText.from_pretrained')
+    @patch('transformers.AutoProcessor.from_pretrained')
+    @patch('vlmeval.vlm.qwen3_vl.model.get_gpu_memory', return_value=[80000])
+    @patch('vlmeval.vlm.qwen3_vl.model.torch.cuda.device_count', return_value=1)
+    def test_all_open_weights_models_pipeline(self, mock_gpu_count, mock_gpu_mem, mock_proc, mock_model):
+        """Test instantiation, prompt generation, and inference pipeline for all open-weights models."""
+        open_weight_models = [
+            "Qwen3.8-27B",
+            "Qwen3.8-27B-Thinking",
+            "Qwen3.8-27B-Instruct",
+            "Qwen3.8-27B-FP8",
+            "Qwen3.8-2.4T-A95B",
+            "Qwen3.8-Flash-Next",
+            "Qwen3.8-Flash-Next-FP8",
+        ]
+
+        mock_processor_instance = MagicMock()
+        mock_processor_instance.apply_chat_template.return_value = "<mock_prompt>"
+        mock_processor_instance.tokenizer.batch_decode.return_value = ["A single red apple."]
+        mock_proc.return_value = mock_processor_instance
+
+        mock_model_instance = MagicMock()
+        mock_model_instance.generate.return_value = [[1, 2, 3, 4]]
+        mock_model.return_value = mock_model_instance
+
+        img_path = os.path.abspath('assets/apple.jpg')
+        test_messages = [
+            {'type': 'image', 'value': img_path},
+            {'type': 'text', 'value': 'Describe what is in this image.'}
+        ]
+
+        for name in open_weight_models:
+            builder = supported_VLM[name]
+            # Override use_vllm=False for testing transformers generation pipeline
+            model = builder(use_vllm=False)
+            model.set_dump_image(lambda item: img_path)
+
+            # Test prompt building for MMMU, MCQ, Y/N, VQA
+            line = {'question': 'Is this an apple?', 'A': 'Yes', 'B': 'No'}
+            mmmu_prompt = model.build_prompt(line, dataset='MMMU_DEV_VAL')
+            self.assertEqual(mmmu_prompt[0]['type'], 'image')
+            self.assertEqual(mmmu_prompt[1]['type'], 'text')
+
+            mcq_prompt = model.build_prompt(line, dataset='MMBench_DEV_EN')
+            self.assertIn('Answer with the option letter only.', mcq_prompt[1]['value'])
+
+            yorn_prompt = model.build_prompt(line, dataset='MME')
+            self.assertIn('Please answer yes or no.', yorn_prompt[1]['value'])
+
+            vqa_prompt = model.build_prompt(line, dataset='DocVQA_VAL')
+            self.assertIn('Please answer concisely', vqa_prompt[1]['value'])
+
+            # Test generation through VLMEvalKit generate() entrypoint
+            with patch('qwen_vl_utils.process_vision_info', return_value=(None, None, None)):
+                out = model.generate(test_messages)
+                self.assertEqual(out, "A single red apple.")
+                print(f"[PASSED] Open-weights model pipeline: {name}")
+
+    @patch('urllib.request.urlopen')
+    def test_all_api_models_pipeline(self, mock_urlopen):
+        """Test instantiation and payload construction for all LMDeploy / vLLM server API models."""
+        api_models = [
+            "Qwen3.8-27B_api",
+            "Qwen3.8-27B_ThinkMode_api",
+            "Qwen3.8-27B_InstructMode_api",
+            "Qwen3.8-2.4T-A95B_api",
+            "Qwen3.8-Flash-Next_api",
+        ]
+
+        for name in api_models:
+            builder = supported_VLM[name]
+            model = builder()
+            self.assertEqual(model.api_base, "http://0.0.0.0:8000/v1/chat/completions")
+            self.assertTrue(hasattr(model, 'generate'))
+            print(f"[PASSED] Server API model configuration: {name}")
+
+    def test_all_dashscope_api_models_pipeline(self):
+        """Test instantiation and message preparation for all DashScope cloud API models."""
+        dashscope_models = [
+            "Qwen3.8-Max",
+            "Qwen3.8-27B-API",
+            "Qwen3.8-Flash-Next-API",
+        ]
+
+        img_path = os.path.abspath('assets/apple.jpg')
+        test_inputs = [
+            {'type': 'image', 'value': img_path},
+            {'type': 'text', 'value': 'What is this?'}
+        ]
+
+        for name in dashscope_models:
+            builder = supported_VLM[name]
+            # Initialize with dummy test key to verify structure
+            model = builder(key='mock-dashscope-key')
+            self.assertTrue(model.is_api)
+            prepared = model._prepare_content(test_inputs)
+            self.assertEqual(prepared[0]['type'], 'image')
+            self.assertEqual(prepared[1]['type'], 'text')
+            print(f"[PASSED] DashScope API model: {name} (target model: {model.model})")
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_qwen3_8.py

@@ -0,0 +1,75 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from vlmeval.config import qwen3_8_series, supported_VLM
+from vlmeval.vlm import Qwen3VLChat
+from vlmeval.vlm.qwen3_vl.model import is_moe_model
+
+
+class TestQwen3_8(unittest.TestCase):
+
+    def test_series_registration(self):
+        self.assertGreater(len(qwen3_8_series), 0)
+        expected_models = [
+            "Qwen3.8-27B",
+            "Qwen3.8-27B-Thinking",
+            "Qwen3.8-27B-Instruct",
+            "Qwen3.8-27B-FP8",
+            "Qwen3.8-2.4T-A95B",
+            "Qwen3.8-Flash-Next",
+            "Qwen3.8-Flash-Next-FP8",
+            "Qwen3.8-27B_api",
+            "Qwen3.8-27B_ThinkMode_api",
+            "Qwen3.8-27B_InstructMode_api",
+            "Qwen3.8-2.4T-A95B_api",
+            "Qwen3.8-Flash-Next_api",
+            "Qwen3.8-Max",
+            "Qwen3.8-27B-API",
+            "Qwen3.8-Flash-Next-API",
+        ]
+        for name in expected_models:
+            self.assertIn(name, qwen3_8_series)
+            self.assertIn(name, supported_VLM)
+
+    def test_moe_detection(self):
+        self.assertTrue(is_moe_model("Qwen/Qwen3.8-2.4T-A95B"))
+        self.assertTrue(is_moe_model("Qwen/Qwen3.8-Flash-Next"))
+        self.assertTrue(is_moe_model("Qwen/Qwen3.8-Flash-Next-FP8"))
+        self.assertFalse(is_moe_model("Qwen/Qwen3.8-27B"))
+        self.assertFalse(is_moe_model("Qwen/Qwen3.8-27B-FP8"))
+
+    @patch('transformers.AutoModelForImageTextToText.from_pretrained')
+    @patch('transformers.AutoProcessor.from_pretrained')
+    @patch('vlmeval.vlm.qwen3_vl.model.get_gpu_memory', return_value=[80000])
+    @patch('vlmeval.vlm.qwen3_vl.model.torch.cuda.device_count', return_value=1)
+    def test_chat_template_kwargs_and_prompts(self, mock_gpu_count, mock_gpu_mem, mock_proc, mock_model):
+        mock_processor_instance = MagicMock()
+        mock_proc.return_value = mock_processor_instance
+
+        # Test initialization with thinking disabled
+        vlm_model = Qwen3VLChat(
+            model_path="Qwen/Qwen3.8-27B",
+            enable_thinking=False,
+            chat_template_kwargs={"custom_flag": True},
+            use_vllm=False
+        )
+        self.assertEqual(vlm_model.chat_template_kwargs, {"custom_flag": True, "enable_thinking": False})
+
+        # Test prompt building for MCQ
+        line = {
+            'question': 'What color is the sky?',
+            'A': 'Blue',
+            'B': 'Green',
+            'image': 'test.jpg'
+        }
+        vlm_model.set_dump_image(lambda item: 'test.jpg')
+        prompt_msgs = vlm_model.build_prompt(line, dataset='MMMU_DEV_VAL')
+        self.assertEqual(len(prompt_msgs), 2)
+        self.assertEqual(prompt_msgs[0]['type'], 'image')
+        self.assertEqual(prompt_msgs[0]['value'], 'test.jpg')
+        self.assertEqual(prompt_msgs[1]['type'], 'text')
+        self.assertIn('What color is the sky?', prompt_msgs[1]['value'])
+
+
+if __name__ == '__main__':
+    unittest.main()

vlmeval/config.py

@@ -1658,6 +1658,182 @@
     ),
 }
 
+qwen3_8_series = {
+    # Open-weights models (vLLM / Transformers)
+    "Qwen3.8-27B": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-27B",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        presence_penalty=1.5,
+        max_new_tokens=32768,
+    ),
+    "Qwen3.8-27B-Thinking": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-27B",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=0.0,
+        repetition_penalty=1.0,
+        max_new_tokens=40960,
+    ),
+    "Qwen3.8-27B-Instruct": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-27B",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        presence_penalty=1.5,
+        repetition_penalty=1.0,
+        chat_template_kwargs={"enable_thinking": False},
+        max_new_tokens=16384,
+    ),
+    "Qwen3.8-27B-FP8": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-27B-FP8",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        presence_penalty=1.5,
+        max_new_tokens=32768,
+    ),
+    "Qwen3.8-2.4T-A95B": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-2.4T-A95B",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=0.0,
+        repetition_penalty=1.0,
+        max_new_tokens=40960,
+    ),
+    "Qwen3.8-Flash-Next": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-Flash-Next",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=1.5,
+        max_new_tokens=32768,
+    ),
+    "Qwen3.8-Flash-Next-FP8": partial(
+        vlm.Qwen3VLChat,
+        model_path="Qwen/Qwen3.8-Flash-Next-FP8",
+        use_custom_prompt=False,
+        use_vllm=True,
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=1.5,
+        max_new_tokens=32768,
+    ),
+
+    # API endpoints (vLLM / LMDeploy server)
+    "Qwen3.8-27B_api": partial(
+        api.LMDeployAPI,
+        model="Qwen/Qwen3.8-27B",
+        api_base="http://0.0.0.0:8000/v1/chat/completions",
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        presence_penalty=1.5,
+        repetition_penalty=1.0,
+        max_new_tokens=32768,
+        retry=6,
+        timeout=1800,
+    ),
+    "Qwen3.8-27B_ThinkMode_api": partial(
+        api.LMDeployAPI,
+        model="Qwen/Qwen3.8-27B",
+        api_base="http://0.0.0.0:8000/v1/chat/completions",
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=0.0,
+        repetition_penalty=1.0,
+        max_tokens=40960,
+        retry=10,
+        timeout=900,
+    ),
+    "Qwen3.8-27B_InstructMode_api": partial(
+        api.LMDeployAPI,
+        model="Qwen/Qwen3.8-27B",
+        api_base="http://0.0.0.0:8000/v1/chat/completions",
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        presence_penalty=1.5,
+        repetition_penalty=1.0,
+        max_tokens=16384,
+        retry=10,
+        timeout=900,
+        chat_template_kwargs={"enable_thinking": False},
+    ),
+    "Qwen3.8-2.4T-A95B_api": partial(
+        api.LMDeployAPI,
+        model="Qwen/Qwen3.8-2.4T-A95B",
+        api_base="http://0.0.0.0:8000/v1/chat/completions",
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=0.0,
+        repetition_penalty=1.0,
+        max_tokens=40960,
+        retry=10,
+        timeout=1800,
+    ),
+    "Qwen3.8-Flash-Next_api": partial(
+        api.LMDeployAPI,
+        model="Qwen/Qwen3.8-Flash-Next",
+        api_base="http://0.0.0.0:8000/v1/chat/completions",
+        temperature=1.0,
+        top_p=0.95,
+        top_k=20,
+        presence_penalty=1.5,
+        repetition_penalty=1.0,
+        max_tokens=32768,
+        retry=10,
+        timeout=1800,
+    ),
+
+    # DashScope Cloud API
+    "Qwen3.8-Max": partial(
+        api.Qwen2VLAPI,
+        model="qwen3.8-max",
+        min_pixels=1280 * 28 * 28,
+        max_pixels=16384 * 28 * 28,
+        max_length=16384,
+    ),
+    "Qwen3.8-27B-API": partial(
+        api.Qwen2VLAPI,
+        model="qwen3.8-27b",
+        min_pixels=1280 * 28 * 28,
+        max_pixels=16384 * 28 * 28,
+        max_length=16384,
+    ),
+    "Qwen3.8-Flash-Next-API": partial(
+        api.Qwen2VLAPI,
+        model="qwen3.8-flash-next",
+        min_pixels=1280 * 28 * 28,
+        max_pixels=16384 * 28 * 28,
+        max_length=16384,
+    ),
+}
+
 sail_series = {
     "SAIL-VL-2B": partial(vlm.SailVL, model_path="BytedanceDouyinContent/SAIL-VL-2B"),
     "SAIL-VL-1.5-2B": partial(vlm.SailVL, model_path="BytedanceDouyinContent/SAIL-VL-1d5-2B", use_msac = True),
@@ -2629,7 +2805,7 @@
     idefics_series, instructblip_series, deepseekvl_series, deepseekvl2_series, deepseekocr_series,
     janus_series, minicpm_series, cogvlm_series, wemm_series, cambrian_series, 
     chameleon_series, video_models, ovis_series, vila_series, mantis_series,
-    mmalaya_series, phi3_series, phi4_series, xgen_mm_series, qwen2vl_series, qwen3vl_series, qwen3_5_series,
+    mmalaya_series, phi3_series, phi4_series, xgen_mm_series, qwen2vl_series, qwen3vl_series, qwen3_5_series, qwen3_8_series,
     slime_series, eagle_series, moondream_series, llama_series, molmo_series,
     kosmos_series, points_series, nvlm_series, vintern_series, h2ovl_series,
     aria_series, smolvlm_series, sail_series, valley_series, vita_series,