Skip to content

Commit 51ef55e

Browse files
RishabhPatilRishabhPatil
committed
[Model] Support Qwen3.8 Series
1 parent bdb6d42 commit 51ef55e

9 files changed

Lines changed: 445 additions & 10 deletions

File tree

‎tests/test_all_qwen3_8_models.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import os
2+
import unittest
3+
from unittest.mock import MagicMock, patch
4+
5+
from vlmeval.config import supported_VLM, qwen3_8_series
6+
from vlmeval.api.adapters import build_adapter
7+
from vlmeval.vlm.qwen3_vl.model import is_moe_model
8+
9+
10+
class TestAllQwen3_8Models(unittest.TestCase):
11+
12+
def test_all_15_models_configuration(self):
13+
"""Verify each of the 15 Qwen 3.8 models has correct configuration in supported_VLM."""
14+
self.assertEqual(len(qwen3_8_series), 15)
15+
for name, partial_func in qwen3_8_series.items():
16+
self.assertIn(name, supported_VLM, f"{name} not found in supported_VLM")
17+
func = partial_func.func
18+
keywords = partial_func.keywords
19+
print(f"Verified config for {name} -> {func.__name__} (keys: {list(keywords.keys())})")
20+
21+
@patch('transformers.AutoModelForImageTextToText.from_pretrained')
22+
@patch('transformers.AutoProcessor.from_pretrained')
23+
@patch('vlmeval.vlm.qwen3_vl.model.get_gpu_memory', return_value=[80000])
24+
@patch('vlmeval.vlm.qwen3_vl.model.torch.cuda.device_count', return_value=1)
25+
def test_all_open_weights_models_pipeline(self, mock_gpu_count, mock_gpu_mem, mock_proc, mock_model):
26+
"""Test instantiation, prompt generation, and inference pipeline for all open-weights models."""
27+
open_weight_models = [
28+
"Qwen3.8-27B",
29+
"Qwen3.8-27B-Thinking",
30+
"Qwen3.8-27B-Instruct",
31+
"Qwen3.8-27B-FP8",
32+
"Qwen3.8-2.4T-A95B",
33+
"Qwen3.8-Flash-Next",
34+
"Qwen3.8-Flash-Next-FP8",
35+
]
36+
37+
mock_processor_instance = MagicMock()
38+
mock_processor_instance.apply_chat_template.return_value = "<mock_prompt>"
39+
mock_processor_instance.tokenizer.batch_decode.return_value = ["A single red apple."]
40+
mock_proc.return_value = mock_processor_instance
41+
42+
mock_model_instance = MagicMock()
43+
mock_output = MagicMock()
44+
mock_model_instance.generate.return_value = [[1, 2, 3, 4]]
45+
mock_model.return_value = mock_model_instance
46+
47+
img_path = os.path.abspath('assets/apple.jpg')
48+
test_messages = [
49+
{'type': 'image', 'value': img_path},
50+
{'type': 'text', 'value': 'Describe what is in this image.'}
51+
]
52+
53+
for name in open_weight_models:
54+
builder = supported_VLM[name]
55+
# Override use_vllm=False for testing transformers generation pipeline
56+
model = builder(use_vllm=False)
57+
model.set_dump_image(lambda l: img_path)
58+
59+
# Test prompt building for MMMU, MCQ, Y/N, VQA
60+
line = {'question': 'Is this an apple?', 'A': 'Yes', 'B': 'No'}
61+
mmmu_prompt = model.build_prompt(line, dataset='MMMU_DEV_VAL')
62+
self.assertEqual(mmmu_prompt[0]['type'], 'image')
63+
self.assertEqual(mmmu_prompt[1]['type'], 'text')
64+
65+
mcq_prompt = model.build_prompt(line, dataset='MMBench_DEV_EN')
66+
self.assertIn('Answer with the option letter only.', mcq_prompt[1]['value'])
67+
68+
yorn_prompt = model.build_prompt(line, dataset='MME')
69+
self.assertIn('Please answer yes or no.', yorn_prompt[1]['value'])
70+
71+
vqa_prompt = model.build_prompt(line, dataset='DocVQA_VAL')
72+
self.assertIn('Please answer concisely', vqa_prompt[1]['value'])
73+
74+
# Test generation through VLMEvalKit generate() entrypoint
75+
with patch('qwen_vl_utils.process_vision_info', return_value=(None, None, None)):
76+
out = model.generate(test_messages)
77+
self.assertEqual(out, "A single red apple.")
78+
print(f"[PASSED] Open-weights model pipeline: {name}")
79+
80+
@patch('urllib.request.urlopen')
81+
def test_all_api_models_pipeline(self, mock_urlopen):
82+
"""Test instantiation and payload construction for all LMDeploy / vLLM server API models."""
83+
api_models = [
84+
"Qwen3.8-27B_api",
85+
"Qwen3.8-27B_ThinkMode_api",
86+
"Qwen3.8-27B_InstructMode_api",
87+
"Qwen3.8-2.4T-A95B_api",
88+
"Qwen3.8-Flash-Next_api",
89+
]
90+
91+
for name in api_models:
92+
builder = supported_VLM[name]
93+
model = builder()
94+
self.assertEqual(model.api_base, "http://0.0.0.0:8000/v1/chat/completions")
95+
self.assertTrue(hasattr(model, 'generate'))
96+
print(f"[PASSED] Server API model configuration: {name}")
97+
98+
def test_all_dashscope_api_models_pipeline(self):
99+
"""Test instantiation and message preparation for all DashScope cloud API models."""
100+
dashscope_models = [
101+
"Qwen3.8-Max",
102+
"Qwen3.8-27B-API",
103+
"Qwen3.8-Flash-Next-API",
104+
]
105+
106+
img_path = os.path.abspath('assets/apple.jpg')
107+
test_inputs = [
108+
{'type': 'image', 'value': img_path},
109+
{'type': 'text', 'value': 'What is this?'}
110+
]
111+
112+
for name in dashscope_models:
113+
builder = supported_VLM[name]
114+
# Initialize with dummy test key to verify structure
115+
model = builder(key='mock-dashscope-key')
116+
self.assertTrue(model.is_api)
117+
prepared = model._prepare_content(test_inputs)
118+
self.assertEqual(prepared[0]['type'], 'image')
119+
self.assertEqual(prepared[1]['type'], 'text')
120+
print(f"[PASSED] DashScope API model: {name} (target model: {model.model})")
121+
122+
123+
if __name__ == '__main__':
124+
unittest.main()

‎tests/test_qwen3_8.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import unittest
2+
from unittest.mock import MagicMock, patch
3+
4+
from vlmeval.api.adapters import build_adapter, Qwen3_8Adapter, Qwen3Adapter
5+
from vlmeval.config import supported_VLM, qwen3_8_series
6+
from vlmeval.vlm import Qwen3_8VLChat, Qwen3VLChat
7+
from vlmeval.vlm.qwen3_vl.model import is_moe_model
8+
9+
10+
class TestQwen3_8(unittest.TestCase):
11+
12+
def test_series_registration(self):
13+
self.assertGreater(len(qwen3_8_series), 0)
14+
expected_models = [
15+
"Qwen3.8-27B",
16+
"Qwen3.8-27B-Thinking",
17+
"Qwen3.8-27B-Instruct",
18+
"Qwen3.8-27B-FP8",
19+
"Qwen3.8-2.4T-A95B",
20+
"Qwen3.8-Flash-Next",
21+
"Qwen3.8-Flash-Next-FP8",
22+
"Qwen3.8-27B_api",
23+
"Qwen3.8-27B_ThinkMode_api",
24+
"Qwen3.8-27B_InstructMode_api",
25+
"Qwen3.8-2.4T-A95B_api",
26+
"Qwen3.8-Flash-Next_api",
27+
"Qwen3.8-Max",
28+
"Qwen3.8-27B-API",
29+
"Qwen3.8-Flash-Next-API",
30+
]
31+
for name in expected_models:
32+
self.assertIn(name, qwen3_8_series)
33+
self.assertIn(name, supported_VLM)
34+
35+
def test_adapter_registration(self):
36+
adapter_38 = build_adapter('qwen3.8')
37+
self.assertIsInstance(adapter_38, Qwen3Adapter)
38+
39+
adapter_38_underscore = build_adapter('qwen3_8')
40+
self.assertIsInstance(adapter_38_underscore, Qwen3Adapter)
41+
42+
payload = {'messages': [{'role': 'user', 'content': 'hi'}]}
43+
adapter_with_pixels = build_adapter('qwen3.8', max_pixels=1000)
44+
processed = adapter_with_pixels.process_payload(payload)
45+
self.assertEqual(processed['mm_processor_kwargs'], {'max_pixels': 1000})
46+
47+
def test_vlm_export(self):
48+
self.assertIs(Qwen3_8VLChat, Qwen3VLChat)
49+
50+
def test_moe_detection(self):
51+
self.assertTrue(is_moe_model("Qwen/Qwen3.8-2.4T-A95B"))
52+
self.assertTrue(is_moe_model("Qwen/Qwen3.8-Flash-Next"))
53+
self.assertTrue(is_moe_model("Qwen/Qwen3.8-Flash-Next-FP8"))
54+
self.assertFalse(is_moe_model("Qwen/Qwen3.8-27B"))
55+
self.assertFalse(is_moe_model("Qwen/Qwen3.8-27B-FP8"))
56+
57+
@patch('transformers.AutoModelForImageTextToText.from_pretrained')
58+
@patch('transformers.AutoProcessor.from_pretrained')
59+
@patch('vlmeval.vlm.qwen3_vl.model.get_gpu_memory', return_value=[80000])
60+
@patch('vlmeval.vlm.qwen3_vl.model.torch.cuda.device_count', return_value=1)
61+
def test_chat_template_kwargs_and_prompts(self, mock_gpu_count, mock_gpu_mem, mock_proc, mock_model):
62+
mock_processor_instance = MagicMock()
63+
mock_proc.return_value = mock_processor_instance
64+
65+
# Test initialization with thinking disabled
66+
vlm_model = Qwen3_8VLChat(
67+
model_path="Qwen/Qwen3.8-27B",
68+
enable_thinking=False,
69+
chat_template_kwargs={"custom_flag": True},
70+
use_vllm=False
71+
)
72+
self.assertEqual(vlm_model.chat_template_kwargs, {"custom_flag": True, "enable_thinking": False})
73+
74+
# Test prompt building for MCQ
75+
line = {
76+
'question': 'What color is the sky?',
77+
'A': 'Blue',
78+
'B': 'Green',
79+
'image': 'test.jpg'
80+
}
81+
vlm_model.set_dump_image(lambda l: 'test.jpg')
82+
prompt_msgs = vlm_model.build_prompt(line, dataset='MMMU_DEV_VAL')
83+
self.assertEqual(len(prompt_msgs), 2)
84+
self.assertEqual(prompt_msgs[0]['type'], 'image')
85+
self.assertEqual(prompt_msgs[0]['value'], 'test.jpg')
86+
self.assertEqual(prompt_msgs[1]['type'], 'text')
87+
self.assertIn('What color is the sky?', prompt_msgs[1]['value'])
88+
89+
90+
if __name__ == '__main__':
91+
unittest.main()

‎vlmeval/api/adapters/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
from .interns1_1 import InternS1_1NoThinkAdapter, InternS1_1ThinkAdapter
44
from .internvl2 import InternVL2Adapter
55
from .internvl3 import InternVL3Adapter
6-
from .qwen3 import Qwen3Adapter
6+
from .qwen3 import Qwen3Adapter, Qwen3_8Adapter
77

88
__all__ = [
99
'ModelAdapter', 'register_adapter', 'build_adapter', 'get_adapter_registry',
1010
'InternVL2Adapter', 'InternVL3Adapter',
1111
'InternS1_1NoThinkAdapter', 'InternS1_1ThinkAdapter',
12-
'CogVLM2Adapter', 'Qwen3Adapter',
12+
'CogVLM2Adapter', 'Qwen3Adapter', 'Qwen3_8Adapter',
1313
]

‎vlmeval/api/adapters/qwen3.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33

44
@register_adapter('qwen3')
5+
@register_adapter('qwen3.5')
6+
@register_adapter('qwen3_5')
7+
@register_adapter('qwen3.8')
8+
@register_adapter('qwen3_8')
59
class Qwen3Adapter(ModelAdapter):
610

711
def __init__(self, max_pixels=None):
@@ -12,3 +16,6 @@ def process_payload(self, payload, dataset=None):
1216
payload = payload.copy()
1317
payload['mm_processor_kwargs'] = {'max_pixels': self.max_pixels}
1418
return payload
19+
20+
21+
Qwen3_8Adapter = Qwen3Adapter

0 commit comments

Comments
 (0)