From 55b9d16c6902ad4f27ed9bf1d0f272ea9fa05bca Mon Sep 17 00:00:00 2001 From: hanchaow Date: Fri, 1 Aug 2025 09:44:49 +0800 Subject: [PATCH 1/8] Add files via upload Add QtuneVL series models, developed by Reconova AI Lab and USTC --- vlmeval/vlm/qtunevl/__init__.py | 4 + vlmeval/vlm/qtunevl/qtune_vl.py | 319 ++++++++++ vlmeval/vlm/qtunevl/qtune_vl_chat.py | 833 +++++++++++++++++++++++++++ 3 files changed, 1156 insertions(+) create mode 100644 vlmeval/vlm/qtunevl/__init__.py create mode 100644 vlmeval/vlm/qtunevl/qtune_vl.py create mode 100644 vlmeval/vlm/qtunevl/qtune_vl_chat.py diff --git a/vlmeval/vlm/qtunevl/__init__.py b/vlmeval/vlm/qtunevl/__init__.py new file mode 100644 index 000000000..b53f2a4ce --- /dev/null +++ b/vlmeval/vlm/qtunevl/__init__.py @@ -0,0 +1,4 @@ +from .qtune_vl import QTuneVL +from .qtune_vl_chat import QTuneVLChat + +__all__ = ['QTuneVL', 'QTuneVLChat'], diff --git a/vlmeval/vlm/qtunevl/qtune_vl.py b/vlmeval/vlm/qtunevl/qtune_vl.py new file mode 100644 index 000000000..bdc301e93 --- /dev/null +++ b/vlmeval/vlm/qtunevl/qtune_vl.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import os +import warnings +import logging + +import torch + +from ..base import BaseModel +from ...smp import listinstr +from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor + + +class QTuneVLPromptMixin: + """ + Mixin class for QTuneVL to build custom prompt for different datasets. + + Requires the following methods to be implemented in the subclass: + - dump_image(line, dataset: str) -> str | list[str] + + Implements the following methods: + - use_custom_prompt(dataset: str) -> bool + - build_prompt(line, dataset: str) -> list[dict[str, str]] + """ + + def __init__(self, *args, use_custom_prompt: bool = True, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._use_custom_prompt = use_custom_prompt + + def set_dump_image(self, dump_image_func): + self.dump_image_func = dump_image_func + + def dump_image(self, line, dataset): + return self.dump_image_func(line) + + def use_custom_prompt(self, dataset: str) -> bool: + from vlmeval.dataset import DATASET_TYPE + dataset_type = DATASET_TYPE(dataset, default=None) + + if not self._use_custom_prompt or listinstr(['MathVista', 'MMMU'], dataset): + return False + if dataset_type == 'MCQ': + if dataset is not None and 'LEGO' in dataset: + return False + return True + if dataset_type == 'Y/N' and dataset in {'HallusionBench', 'POPE'}: # MME has it's own prompt + return True + if dataset_type == 'VQA' and dataset not in {'MMVet'}: # MMVet VQA has it's own prompt + return True + return False + + def build_prompt(self, line, dataset: str) -> list[dict[str, str]]: + from vlmeval.dataset import DATASET_TYPE + + dataset_type = DATASET_TYPE(dataset, default=None) + if dataset_type == 'MCQ': + return self._build_mcq_prompt(line, dataset) + if dataset_type == 'Y/N': + return self._build_yorn_prompt(line, dataset) + if dataset_type == 'VQA': + return self._build_vqa_prompt(line, dataset) + raise ValueError(f'Unsupported dataset: {dataset}') + + def _build_mcq_prompt(self, line, dataset: str) -> list[dict[str, str]]: + """change the prompt for MCQ dataset: use chinese prompt if the question contains chinese characters.""" + MCQ_CN_PROMPT = '请直接回答选项字母。' + MCQ_EN_PROMPT = 'Please select the correct answer from the options above.' + + import string + + import pandas as pd + + def cn_string(s): + import re + + if re.search('[\u4e00-\u9fff]', s): + return True + return False + + tgt_path = self.dump_image(line, dataset) + question = line['question'] + options = {cand: line[cand] for cand in string.ascii_uppercase if cand in line and not pd.isna(line[cand])} + options_prompt = 'Options:\n' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + prompt = '' + if hint is not None: + prompt += f'Hint: {hint}\n' + prompt += f'Question: {question}\n' + if len(options): + prompt += options_prompt + prompt += MCQ_CN_PROMPT if cn_string(prompt) else MCQ_EN_PROMPT + prompt = prompt.rstrip() + msgs = [] + if isinstance(tgt_path, list): + msgs.extend([dict(type='image', value=p) for p in tgt_path]) + else: + msgs = [dict(type='image', value=tgt_path)] + msgs.append(dict(type='text', value=prompt)) + return msgs + + def _build_yorn_prompt(self, line, dataset: str) -> list[dict[str, str]]: + """change the prompt for YORN dataset:""" + YORN_PROMPT = ' Please answer yes or no.' + + tgt_path = self.dump_image(line, dataset) + question = line['question'] + msgs = [] + if isinstance(tgt_path, list): + msgs.extend([dict(type='image', value=p) for p in tgt_path]) + else: + msgs = [dict(type='image', value=tgt_path)] + msgs.append(dict(type='text', value=question)) + assert msgs[-1]['type'] == 'text' + msgs[-1]['value'] += YORN_PROMPT + return msgs + + def _build_vqa_prompt(self, line, dataset: str) -> list[dict[str, str]]: + """change the prompt for VQA dataset:""" + VQA_PROMPT = '\nPlease try to answer the question with short words or phrases if possible.' + + tgt_path = self.dump_image(line, dataset) + question = line['question'] + msgs = [] + if isinstance(tgt_path, list): + msgs.extend([dict(type='image', value=p) for p in tgt_path]) + else: + msgs = [dict(type='image', value=tgt_path)] + msgs.append(dict(type='text', value=question)) + assert msgs[-1]['type'] == 'text' + msgs[-1]['value'] += VQA_PROMPT + return msgs + + +def ensure_image_url(image: str) -> str: + prefixes = ['http://', 'https://', 'file://', 'data:image;'] + if any(image.startswith(prefix) for prefix in prefixes): + return image + if os.path.exists(image): + return 'file://' + image + raise ValueError(f'Invalid image: {image}') + + +def ensure_video_url(video: str) -> str: + prefixes = ['http://', 'https://', 'file://', 'data:video;'] + if any(video.startswith(prefix) for prefix in prefixes): + return video + if os.path.exists(video): + return 'file://' + video + raise ValueError(f'Invalid video: {video}') + + +class QTuneVL(QTuneVLPromptMixin, BaseModel): + INSTALL_REQ = False + INTERLEAVE = True + VIDEO_LLM = True + + def __init__( + self, + model_path: str, + min_pixels: int | None = None, + max_pixels: int | None = None, + total_pixels: int | None = None, + max_new_tokens=2048, + top_p=0.001, + top_k=1, + temperature=0.01, + repetition_penalty=1.0, + use_custom_prompt: bool = True, + system_prompt: str | None = None, + post_process: bool = False, # if True, will try to only extract stuff in the last \boxed{}. + verbose: bool = False, + **kwargs, + ): + super().__init__(use_custom_prompt=use_custom_prompt) + self.min_pixels = min_pixels + self.max_pixels = max_pixels + self.total_pixels = total_pixels + self.max_new_tokens = max_new_tokens + if self.total_pixels and self.total_pixels > 24576 * 28 * 28: + print('The total number of video tokens might become too large, resulting in an overly long input sequence. We recommend lowering **total_pixels** to below **24576 × 28 × 28**.') # noqa: E501 + self.generate_kwargs = dict( + max_new_tokens=self.max_new_tokens, + top_p=top_p, + top_k=top_k, + temperature=temperature, + repetition_penalty=repetition_penalty, + ) + self.system_prompt = system_prompt + self.verbose = verbose + self.post_process = post_process + self.fps = kwargs.pop('fps', 2) + self.nframe = kwargs.pop('nframe', 128) + if self.fps is None and self.nframe is None: + print("Warning: fps and nframe are both None, \ + using default nframe/fps setting in qwen-vl-utils/qwen-omni-utils, \ + the fps/nframe setting in video dataset is omitted") + self.FRAME_FACTOR = 2 + assert model_path is not None + self.model_path = model_path + self.processor = AutoProcessor.from_pretrained(model_path) + + self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_path, torch_dtype='auto', device_map="auto", attn_implementation='flash_attention_2' + ) + self.model.eval() + + torch.cuda.empty_cache() + + def _prepare_content(self, inputs: list[dict[str, str]], dataset: str | None = None) -> list[dict[str, str]]: + """ + inputs list[dict[str, str]], each dict has keys: ['type', 'value'] + """ + content = [] + for s in inputs: + if s['type'] == 'image': + item = {'type': 'image', 'image': ensure_image_url(s['value'])} + if dataset == 'OCRBench': + item['min_pixels'] = 10 * 10 * 28 * 28 + warnings.warn(f"OCRBench dataset uses custom min_pixels={item['min_pixels']}") + if self.max_pixels is not None: + item['max_pixels'] = self.max_pixels + else: + if self.min_pixels is not None: + item['min_pixels'] = self.min_pixels + if self.max_pixels is not None: + item['max_pixels'] = self.max_pixels + if self.total_pixels is not None: + item['total_pixels'] = self.total_pixels + elif s['type'] == 'video': + item = { + 'type': 'video', + 'video': ensure_video_url(s['value']) + } + if self.min_pixels is not None: + item['min_pixels'] = self.min_pixels + if self.max_pixels is not None: + item['max_pixels'] = self.max_pixels + if self.total_pixels is not None: + item['total_pixels'] = self.total_pixels + if self.fps is not None: + item['fps'] = self.fps + elif self.nframe is not None: + import cv2 + video = cv2.VideoCapture(s['value']) + frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + video.release() + if frame_count < self.nframe: + new_frame_count = frame_count // self.FRAME_FACTOR * self.FRAME_FACTOR + print(f"use {new_frame_count} for {s['value']}") + item['nframes'] = new_frame_count + else: + item['nframes'] = self.nframe + elif s['type'] == 'text': + item = {'type': 'text', 'text': s['value']} + elif s['type'] == 'audio': + item = {'type':'audio','audio':s['value']} + else: + raise ValueError(f"Invalid message type: {s['type']}, {s}") + content.append(item) + return content + + def generate_inner_transformers(self, message, dataset=None): + try: + from qwen_vl_utils import process_vision_info + except Exception as err: + logging.critical("qwen_vl_utils not found, please install it via 'pip install qwen-vl-utils'") # noqa: E501 + raise err + + messages = [] + if self.system_prompt is not None: + messages.append({'role': 'system', 'content': self.system_prompt}) + messages.append({'role': 'user', 'content': self._prepare_content(message, dataset=dataset)}) + if self.verbose: + print(f'\033[31m{messages}\033[0m') + + text = self.processor.apply_chat_template([messages], tokenize=False, add_generation_prompt=True) + + images, videos = process_vision_info([messages]) + inputs = self.processor(text=text, images=images, videos=videos, padding=True, return_tensors='pt') # noqa: E501 + inputs = inputs.to('cuda') + + generated_ids = self.model.generate( + **inputs, + **self.generate_kwargs, + ) + generated_ids = [ + output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, generated_ids) + ] + out = self.processor.tokenizer.batch_decode( + generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + response = out[0] + if self.post_process: + resp = response.split('\\boxed{')[-1] + lt = len(resp) + counter, end = 1, None + for i in range(lt): + if resp[i] == '{': + counter += 1 + elif resp[i] == '}': + counter -= 1 + if counter == 0: + end = i + break + elif i == lt - 1: + end = lt + break + if end is not None: + response = resp[:end] + + if self.verbose: + print(f'\033[32m{response}\033[0m') + return response + + def generate_inner(self, message, dataset=None): + return self.generate_inner_transformers(message, dataset=dataset) + diff --git a/vlmeval/vlm/qtunevl/qtune_vl_chat.py b/vlmeval/vlm/qtunevl/qtune_vl_chat.py new file mode 100644 index 000000000..2e9ed0d4c --- /dev/null +++ b/vlmeval/vlm/qtunevl/qtune_vl_chat.py @@ -0,0 +1,833 @@ +import math +import pandas as pd +import random +import re +import string +import torch +import torch.distributed as dist +import torchvision.transforms as T +import transformers +import warnings +from PIL import Image +from torchvision.transforms.functional import InterpolationMode +from transformers import AutoTokenizer, AutoConfig, AutoModel, CLIPImageProcessor + +from ..base import BaseModel +from ...dataset import DATASET_TYPE, DATASET_MODALITY +from ...smp import * + + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +def build_transform(input_size): + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD + transform = T.Compose([ + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img), + T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=MEAN, std=STD) + ]) + return transform + + +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): + best_ratio_diff = float('inf') + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + +###################################################################################### +def preprocess_padding_global(image, net_input_size=448): + # 不形变的resize方式 + orig_width, orig_height = image.size + if orig_height > orig_width: + scale_ratio = net_input_size / orig_height + scale_h = net_input_size + scale_w = int(orig_width * scale_ratio) + else: + scale_ratio = net_input_size / orig_width + scale_h = int(orig_height * scale_ratio) + scale_w = net_input_size + + resized = image.resize((scale_w, scale_h), Image.BILINEAR) + new_image = Image.new('RGB', (net_input_size, net_input_size), (0, 0, 0)) + paste_x = (net_input_size - scale_w) // 2 + paste_y = (net_input_size - scale_h) // 2 + new_image.paste(resized, (paste_x, paste_y)) + return new_image + +def dynamic_preprocess(image, min_num=5, max_num=6, image_size=448, use_thumbnail=False): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + padding_global = preprocess_padding_global(image, image_size) + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if + i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + processed_images.append(padding_global) + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images, target_aspect_ratio + + +def dynamic_preprocess2(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False, prior_aspect_ratio=None): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if + i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + new_target_ratios = [] + if prior_aspect_ratio is not None: + for i in target_ratios: + if prior_aspect_ratio[0] % i[0] != 0 or prior_aspect_ratio[1] % i[1] != 0: + new_target_ratios.append(i) + else: + continue + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, new_target_ratios, orig_width, orig_height, image_size) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +def load_image(image_file, input_size=448, max_num=6, upscale=False): + image = Image.open(image_file).convert('RGB') + if upscale: + image = image.resize((image.width * 2, image.height * 2), Image.BILINEAR) + transform = build_transform(input_size=input_size) + images, target_aspect_ratio = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num) + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values, target_aspect_ratio + + +def load_image2(image_file, input_size=448, target_aspect_ratio=(1, 1), min_num=1, max_num=6): + image = Image.open(image_file).convert('RGB') + transform = build_transform(input_size=input_size) + images = dynamic_preprocess2( + image, + image_size=input_size, + prior_aspect_ratio=target_aspect_ratio, + use_thumbnail=True, + min_num=min_num, + max_num=max_num) + + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values + + +def dynamic_preprocess3(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if + i * j <= max_num and i * j >= min_num) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +def load_image3(image_file, input_size=448, max_num=6, upscale=False): + image = Image.open(image_file).convert('RGB') + if upscale: + image = image.resize((image.width * 2, image.height * 2), Image.BILINEAR) + transform = build_transform(input_size=input_size) + images = dynamic_preprocess3(image, image_size=input_size, use_thumbnail=True, max_num=max_num) + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values +###################################################################################### + + +def get_local_rank_and_local_world_size(): + if not dist.is_available(): + return 0, 1 + if not dist.is_initialized(): + return 0, 1 + + if 'SLURM_LOCALID' in os.environ: + local_rank = int(os.environ['SLURM_LOCALID']) + local_world_size = int(os.environ['SLURM_NTASKS_PER_NODE']) + return local_rank, local_world_size + + if 'LOCAL_RANK' in os.environ and 'LOCAL_WORLD_SIZE' in os.environ: + return int(os.environ['LOCAL_RANK']), int(os.environ['LOCAL_WORLD_SIZE']) + + raise NotImplementedError( + "Fail to get local_rank and local_world_size! " + "Please ensure that you set the environment variable " + "`LOCAL_RANK` and `LOCAL_WORLD_SIZE`" + ) + + +def split_model(model_path): + num_gpus_per_node = torch.cuda.device_count() + rank, world_size = get_rank_and_world_size() + try: + local_rank, local_world_size = get_local_rank_and_local_world_size() + except: + local_rank = rank + + if 'GPUS_PER_PROCESS' in os.environ: + gpus_per_process = int(os.environ['GPUS_PER_PROCESS']) + else: + gpus_per_process = 8 # default to use 8 GPUs for one model + gpus_per_process = min(gpus_per_process, num_gpus_per_node // local_world_size) + start_gpu = local_rank * gpus_per_process + end_gpu = start_gpu + gpus_per_process + + assert end_gpu <= num_gpus_per_node, f"Process {local_rank} tries to access GPU {end_gpu}, " \ + f"but only {num_gpus_per_node} GPUs are available per node." + + visible_devices = list(range(start_gpu, end_gpu)) + + device_map = {} + config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + + num_gpus_for_vit = 0.5 + num_layers = config.llm_config.num_hidden_layers + num_layers_per_gpu = math.ceil(num_layers / (len(visible_devices) - num_gpus_for_vit)) + num_layers_per_gpu = [num_layers_per_gpu] * len(visible_devices) + num_layers_per_gpu[0] = math.ceil(num_layers_per_gpu[0] * 0.5) + + layer_cnt = 0 + for i, num_layer in enumerate(num_layers_per_gpu): + for j in range(num_layer): + device_map[f'language_model.model.layers.{layer_cnt}'] = visible_devices[i] + layer_cnt += 1 + device_map['vision_model'] = visible_devices[0] + device_map['mlp1'] = visible_devices[0] + device_map['language_model.model.tok_embeddings'] = visible_devices[0] + device_map['language_model.model.embed_tokens'] = visible_devices[0] + device_map['language_model.output'] = visible_devices[0] + device_map['language_model.model.norm'] = visible_devices[0] + device_map['language_model.model.rotary_emb'] = visible_devices[0] + device_map['language_model.lm_head'] = visible_devices[0] + device_map[f'language_model.model.layers.{num_layers - 1}'] = visible_devices[0] + + return device_map, visible_devices + + +def build_mcq_cot_prompt(line, prompt): + cot_prompt = ( + "Answer the preceding multiple choice question. The last line of your response should follow " + "this format: 'Answer: \\boxed{$LETTER}' (without quotes), where LETTER is one of the options. " + "If you are uncertain or the problem is too complex, make a reasoned guess based on the " + "information provided. Avoid repeating steps indefinitely—provide your best guess even if " + "unsure. Think step by step logically, considering all relevant information before answering." + ) + prompt = prompt.replace("Answer with the option's letter from the given choices directly.", '').strip() + prompt = prompt + '\n' + cot_prompt + + return prompt + + +def build_qa_cot_prompt(line, prompt): + cot_prompt = ( + "Answer the preceding question. The last line of your response should follow this format: " + "'Answer: \\boxed{$FINAL_ANSWER}' (without quotes), where 'FINAL_ANSWER' is your conclusion " + "based on the reasoning provided. If you are uncertain or the problem is too complex, make " + "a reasoned guess based on the information provided. Avoid repeating steps indefinitely—" + "provide your best guess even if unsure. Think step by step logically, considering all " + "relevant information before answering." + ) + prompt = prompt + '\n' + cot_prompt + + return prompt + + +def build_multi_choice_prompt(line, dataset=None): + question = line['question'] + hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None + if hint is not None: + question = hint + '\n' + question + + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + for key, item in options.items(): + question += f'\n{key}. {item}' + prompt = question + + if len(options): + prompt += '\n请直接回答选项字母。' if cn_string( + prompt) else "\nAnswer with the option's letter from the given choices directly." + else: + prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.' + + return prompt + + +def build_video_prompt(prompt, dataset=None, max_frames=64): + for start in range(0, max_frames, 8): + images_to_remove = ''.join([f'' for i in range(start + 1, start + 9)]) + prompt = prompt.replace(images_to_remove, '') + for i in range(max_frames): + prompt = prompt.replace(f'Image-{i + 1}', f'Frame-{i + 1}') + if listinstr(['MMBench-Video'], dataset): + prompt = prompt.replace('\nAnswer:', '') + elif listinstr(['Video-MME', 'WorldSense'], dataset): + prompt = prompt.replace('\nAnswer:', '') + prompt += "\nAnswer with the option's letter from the given choices directly." + elif listinstr(['MVBench'], dataset): + prompt = prompt.replace('Best option:(', '') + + return prompt + + +def reorganize_prompt(message, image_num, dataset=None): + if dataset is not None and listinstr(['MUIRBench'], dataset): + prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text']) + images_to_remove = ' '.join([''] * image_num) + prompt = prompt.replace(images_to_remove, '') + for i in range(image_num): + prompt = prompt.replace('', f'', 1) + prompt = ''.join([f'Image-{i + 1}: \n' for i in range(image_num)]) + prompt + elif image_num == 1: + prompt = '\n' + '\n'.join([x['value'] for x in message if x['type'] == 'text']) + else: + prompt, image_idx = '', 1 + for x in message: + if x['type'] == 'text': + prompt += x['value'] + elif x['type'] == 'image': + prompt += f'' + image_idx += 1 + prompt = ''.join([f'Image-{i + 1}: \n' for i in range(image_num)]) + prompt + images_to_remove = ''.join([f'' for i in range(image_num)]) + prompt = prompt.replace(images_to_remove, '') + return prompt + + +mpo_prompt_with_final_answer = ( + "Your task is to answer the question below. " + "Give step by step reasoning before you answer, and when you're ready to answer, " + "please use the format \"Final answer: ..\"" + "\n\n" + "Question:" + "\n\n" + "{question}" +) + +mpo_prompt_without_final_answer = ( + "Your task is to answer the question below. " + "Give step by step reasoning. " + "\n\n" + "Question:" + "\n\n" + "{question}" +) + + +def mpo_post_processing(response, dataset): + + def extract_answer(text): + match = re.search(r'(Final answer:|Answer:)\s*(.*)', text, re.IGNORECASE) + if match: + return match.group(2).strip() + return text + + if dataset is not None and (DATASET_TYPE(dataset) in ['Y/N', 'MCQ'] or listinstr(['CRPE'], dataset)): + response = extract_answer(response).strip() + return response + + +def build_mpo_prompt(message, line, dataset): + if listinstr(['LLaVABench', 'MMVet'], dataset): + return message + + question_orig = line['question'] + if listinstr(['MathVerse', 'MathVision'], dataset): + question_orig = question_orig.split('Question:', 1)[-1].strip() + question_orig = question_orig.replace('Choices:\n', '').strip() + if listinstr(['WeMath'], dataset): + question_orig = question_orig.replace('Regarding the format, please answer following the template below, and be sure to include two <> symbols:\n: <> : <>', '').strip() # noqa: E501 + options = { + cand: line[cand] + for cand in string.ascii_uppercase + if cand in line and not pd.isna(line[cand]) + } + options_prompt = '' + for key, item in options.items(): + options_prompt += f'{key}. {item}\n' + + if options_prompt.strip(): + question_orig = f'{question_orig}\n{options_prompt}' + + cot_prompt = mpo_prompt_with_final_answer + prompt = cot_prompt.format(question=question_orig).strip() + message[0]['value'] = prompt + return message + + +class QTuneVLChat(BaseModel): + INSTALL_REQ = False + INTERLEAVE = True + + def __init__(self, + model_path='hanchaow/QTuneVL1-2B', + load_in_8bit=False, + use_mpo_prompt=False, + version='V1.0', + # Best-of-N parameters + best_of_n=1, + reward_model_path=None, + **kwargs): + + assert best_of_n >= 1 + assert model_path is not None + assert version_cmp(transformers.__version__, '4.37.2', 'ge') + + self.use_mpo_prompt = use_mpo_prompt + self.use_cot = (os.getenv('USE_COT') == '1') + + self.model_path = model_path + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False) + + # Regular expression to match the pattern 'Image' followed by a number, e.g. Image1 + self.pattern = r'Image(\d+)' + # Replacement pattern to insert a hyphen between 'Image' and the number, e.g. Image-1 + self.replacement = r'Image-\1' + + # Convert QtuneVL2 response to dataset format + # e.g. Image1 -> Image-1 + + # Regular expression to match the pattern 'Image-' followed by a number + self.reverse_pattern = r'Image-(\d+)' + # Replacement pattern to remove the hyphen (Image-1 -> Image1) + self.reverse_replacement = r'Image\1' + + if auto_split_flag(): + device_map, visible_devices = split_model(model_path=model_path) + self.device = visible_devices[0] + self.model = AutoModel.from_pretrained( + model_path, + torch_dtype=torch.bfloat16, + load_in_8bit=load_in_8bit, + trust_remote_code=True, + low_cpu_mem_usage=True, + device_map=device_map).eval() + else: + self.model = AutoModel.from_pretrained( + model_path, + torch_dtype=torch.bfloat16, + load_in_8bit=load_in_8bit, + trust_remote_code=True, + low_cpu_mem_usage=True).eval().cuda() + self.device = 'cuda' + + if best_of_n > 1: + assert reward_model_path is not None + + if auto_split_flag(): + rm_device_map, visible_devices = split_model(model_path=reward_model_path) + rm_kwargs = {'device_map': rm_device_map} + else: + rm_kwargs = {} + + self.reward_tokenizer = AutoTokenizer.from_pretrained( + reward_model_path, trust_remote_code=True, use_fast=False + ) + self.reward_model = AutoModel.from_pretrained( + reward_model_path, + torch_dtype=torch.bfloat16, + load_in_8bit=load_in_8bit, + trust_remote_code=True, + low_cpu_mem_usage=True, **rm_kwargs).eval() + + if not auto_split_flag(): + self.reward_model = self.reward_model.to(self.device) + + if not self.use_cot: + os.environ['USE_COT'] = '1' + self.use_cot = True + print('[Warning] Since Best-of-N is enabled, USE_COT is forced to be set to 1.') + + print(f'Enable Best-of-N evaluation with PRM: {reward_model_path}') + + self.image_size = self.model.config.vision_config.image_size + self.version = version + self.best_of_n = best_of_n + kwargs_default = dict(do_sample=False, max_new_tokens=4096, top_p=None) + kwargs_default.update(kwargs) + self.kwargs = kwargs_default + + warnings.warn(f'Following kwargs received: {self.kwargs}, will use as generation config. ') + + def use_custom_prompt(self, dataset): + assert dataset is not None + if listinstr(['MMDU', 'MME-RealWorld', 'MME-RealWorld-CN', 'WeMath_COT', 'MMAlignBench'], dataset): + # For Multi-Turn we don't have custom prompt + return False + if DATASET_MODALITY(dataset) == 'VIDEO': + # For Video benchmarks we don't have custom prompt at here + return False + else: + return True + + def build_prompt(self, line, dataset=None): + use_mpo_prompt = self.use_mpo_prompt and (self.use_cot or dataset in ['MMStar', 'HallusionBench', 'OCRBench']) + + assert self.use_custom_prompt(dataset) + assert dataset is None or isinstance(dataset, str) + tgt_path = self.dump_image(line, dataset) + + if dataset is not None and DATASET_TYPE(dataset) == 'Y/N': + question = line['question'] + if listinstr(['MME'], dataset): + prompt = question + ' Answer the question using a single word or phrase.' + elif listinstr(['HallusionBench', 'AMBER'], dataset): + prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.' + else: + prompt = question + elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ': + prompt = build_multi_choice_prompt(line, dataset) + if os.getenv('USE_COT') == '1': + prompt = build_mcq_cot_prompt(line, prompt) + elif dataset is not None and DATASET_TYPE(dataset) == 'VQA': + question = line['question'] + if listinstr(['LLaVABench', 'WildVision'], dataset): + prompt = question + '\nAnswer this question in detail.' + elif listinstr(['OCRVQA', 'TextVQA', 'ChartQA', 'DocVQA', 'InfoVQA', 'OCRBench', + 'DUDE', 'SLIDEVQA', 'GQA', 'MMLongBench_DOC'], dataset): + prompt = question + '\nAnswer the question using a single word or phrase.' + elif listinstr(['MathVista', 'MathVision', 'VCR', 'MTVQA', 'MMVet', 'MathVerse', + 'MMDU', 'CRPE', 'MIA-Bench', 'MM-Math', 'DynaMath', 'QSpatial', + 'WeMath', 'LogicVista'], dataset): + prompt = question + if os.getenv('USE_COT') == '1': + prompt = build_qa_cot_prompt(line, prompt) + else: + prompt = question + '\nAnswer the question using a single word or phrase.' + else: + # VQA_ex_prompt: OlympiadBench, VizWiz + prompt = line['question'] + if os.getenv('USE_COT') == '1': + prompt = build_qa_cot_prompt(line, prompt) + + message = [dict(type='text', value=prompt)] + message.extend([dict(type='image', value=s) for s in tgt_path]) + + if use_mpo_prompt: + message = build_mpo_prompt(message, line, dataset) + return message + + def set_max_num(self, dataset): + # The total limit on the number of images processed, set to avoid Out-of-Memory issues. + self.total_max_num = 64 + if dataset is None: + self.max_num = 6 + return None + res_12_datasets = ['ChartQA_TEST', 'MMMU_DEV_VAL', 'MMMU_TEST', 'MME-RealWorld', + 'VCR_EN', 'VCR_ZH', 'OCRVQA'] + res_18_datasets = ['DocVQA_VAL', 'DocVQA_TEST', 'DUDE', 'MMLongBench_DOC', 'SLIDEVQA'] + res_24_datasets = ['InfoVQA_VAL', 'InfoVQA_TEST', 'OCRBench', 'HRBench4K', 'HRBench8K'] + if DATASET_MODALITY(dataset) == 'VIDEO': + self.max_num = 1 + elif listinstr(res_12_datasets, dataset): + self.max_num = 12 + elif listinstr(res_18_datasets, dataset): + self.max_num = 18 + elif listinstr(res_24_datasets, dataset): + self.max_num = 24 + else: + self.max_num = 6 + + @torch.no_grad() + def generate_v2(self, message, dataset=None): + + use_mpo_prompt = self.use_mpo_prompt and (self.use_cot or dataset in ['MMStar', 'HallusionBench', 'OCRBench']) + + image_num = len([x for x in message if x['type'] == 'image']) + max_num = max(1, min(self.max_num, self.total_max_num // image_num)) + prompt = reorganize_prompt(message, image_num, dataset=dataset) + + if dataset is not None and DATASET_MODALITY(dataset) == 'VIDEO': + prompt = build_video_prompt(prompt, dataset) + + if image_num > 1: + image_path = [x['value'] for x in message if x['type'] == 'image'] + num_patches_list, pixel_values_list = [], [] + for image_idx, file_name in enumerate(image_path): + + if dataset is not None and (listinstr(['MMBench'], dataset) or + listinstr(['MMStar'], dataset) or listinstr(['MMVet'], dataset)): + upscale_flag = image_idx == 0 and dataset is not None and listinstr(['MMMU'], dataset) + curr_pixel_values = load_image3( + file_name, max_num=max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) + num_patches_list.append(curr_pixel_values.size(0)) + pixel_values_list.append(curr_pixel_values) + else: + upscale_flag = image_idx == 0 and dataset is not None and listinstr(['MMMU'], dataset) + curr_pixel_values, target_aspect_ratio = load_image( + file_name, max_num=max_num, upscale=upscale_flag) + curr_pixel_values = curr_pixel_values.cuda().to(torch.bfloat16) + curr_pixel_values2 = load_image2( + file_name, target_aspect_ratio=target_aspect_ratio, max_num=max_num) + curr_pixel_values2 = curr_pixel_values2.cuda().to(torch.bfloat16) + if (listinstr(['MathVista'], dataset) or + listinstr(['HallusionBench'], dataset) or listinstr(['OCRBench'], dataset)): + curr_pixel_values = torch.cat( + (curr_pixel_values[:-1], curr_pixel_values2[:-1], curr_pixel_values[-1:]), 0) + else: + curr_pixel_values = torch.cat( + (curr_pixel_values2[:-1], curr_pixel_values[-1:]), 0) + num_patches_list.append(curr_pixel_values.size(0)) + pixel_values_list.append(curr_pixel_values) + pixel_values = torch.cat(pixel_values_list, dim=0) + elif image_num == 1: + + + if dataset is not None and (listinstr(['MMBench'], dataset) or + listinstr(['MMStar'], dataset) or listinstr(['MMVet'], dataset)): + image_path = [x['value'] for x in message if x['type'] == 'image'][0] + upscale_flag = dataset is not None and listinstr(['MMMU'], dataset) + pixel_values = load_image3( + image_path, max_num=max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) + num_patches_list = [pixel_values.size(0)] + else: + image_path = [x['value'] for x in message if x['type'] == 'image'][0] + upscale_flag = dataset is not None and listinstr(['MMMU'], dataset) + pixel_values, target_aspect_ratio = load_image( + image_path, max_num=max_num, upscale=upscale_flag) + pixel_values = pixel_values.to(self.device).to(torch.bfloat16) + pixel_values2 = load_image2( + image_path, target_aspect_ratio=target_aspect_ratio, max_num=max_num) + pixel_values2 = pixel_values2.cuda().to(torch.bfloat16) + if (listinstr(['MathVista'], dataset) or + listinstr(['HallusionBench'], dataset) or listinstr(['OCRBench'], dataset)): + pixel_values = torch.cat((pixel_values[:-1], pixel_values2[:-1], pixel_values[-1:]), 0) + else: + pixel_values = torch.cat((pixel_values2[:-1], pixel_values[-1:]), 0) + num_patches_list = [pixel_values.size(0)] + else: + pixel_values = None + num_patches_list = [] + + response_list = [] + for idx in range(self.best_of_n): + kwargs_default = self.kwargs.copy() + kwargs_default['do_sample'] = idx > 0 + kwargs_default['temperature'] = 0.7 + kwargs_default['top_p'] = 0.95 + + response = self.model.chat( + self.tokenizer, + pixel_values=pixel_values, + num_patches_list=num_patches_list, + question=prompt, + generation_config=kwargs_default, + verbose=idx == 0, + ) + response_list.append(response) + + if self.best_of_n > 1: + response_list = self.reward_model.select_best_response( + tokenizer=self.reward_tokenizer, + question=prompt, + response_list=response_list, + pixel_values=pixel_values, + num_patches_list=num_patches_list, + ) + response = response_list[0] + + if use_mpo_prompt: + response = mpo_post_processing(response, dataset) + return response + + def generate_inner(self, message, dataset=None): + self.set_max_num(dataset) + print(f'QtuneVL model version: {self.version}') + if self.version in ['V1.0', 'V1.5']: + return self.generate_v2(message, dataset) + else: + raise ValueError(f'Unsupported version: {self.version}') + + def build_history(self, message): + # Global Variables + image_path = [] + image_cnt = 0 + + def concat_tilist(tilist): + nonlocal image_cnt # Declare image_cnt as nonlocal to modify it + prompt = '' + for item in tilist: + # Substitute the pattern in the text + if item['type'] == 'text': + prompt += re.sub(self.pattern, self.replacement, item['value']) + elif item['type'] == 'image': + image_cnt += 1 + prompt += '\n' + image_path.append(item['value']) + return prompt + + # Only previous messages + assert len(message) % 2 == 0 + history = [] + for i in range(len(message) // 2): + m1, m2 = message[2 * i], message[2 * i + 1] + assert m1['role'] == 'user' and m2['role'] == 'assistant' + history.append((concat_tilist(m1['content']), concat_tilist(m2['content']))) + + return history, image_path, image_cnt + + def chat_inner_v2(self, message, dataset=None): + + if len(message) > 1: + history, image_path, image_cnt = self.build_history(message[:-1]) + else: + history, image_path, image_cnt = None, [], 1 + current_msg = message[-1] + question = '' + + # If message is just text in the conversation + if len(current_msg['content']) == 1 and current_msg['content'][0]['type'] == 'text': + question = current_msg['content'][0]['value'] + question = re.sub(self.pattern, self.replacement, question) # Fix pattern as per QtuneVL + else: + for msg in current_msg['content']: + if msg['type'] == 'text': + question += re.sub(self.pattern, self.replacement, msg['value']) + elif msg['type'] == 'image': + image_cnt += 1 + question += '\n' + image_path.append(msg['value']) + + if image_cnt > 1: + num_patches_list = [] + pixel_values_list = [] + for image_idx, file_name in enumerate(image_path): + upscale_flag = image_idx == 0 and dataset is not None and listinstr(['MMMU_DEV_VAL'], dataset) + curr_pixel_values = load_image( + file_name, max_num=self.max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) + num_patches_list.append(curr_pixel_values.size(0)) + pixel_values_list.append(curr_pixel_values) + pixel_values = torch.cat(pixel_values_list, dim=0) + elif image_cnt == 1: + upscale_flag = listinstr(['MMMU_DEV_VAL'], dataset) + pixel_values = load_image( + image_path, max_num=self.max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) + num_patches_list = [pixel_values.size(0)] + else: + pixel_values = None + num_patches_list = [] + + response, history = self.model.chat( + self.tokenizer, + pixel_values=pixel_values, + num_patches_list=num_patches_list, + question=question, + generation_config=self.kwargs, + history=history, + return_history=True + ) + + response = re.sub(self.reverse_pattern, self.reverse_replacement, response) + + return response + + def chat_inner(self, message, dataset=None): + self.set_max_num(dataset) + + if self.version in ['V1.0', 'V1.5']: + kwargs_default = dict(do_sample=False, max_new_tokens=512, top_p=None, num_beams=1) + self.kwargs = kwargs_default + return self.chat_inner_v2(message, dataset) + else: + raise ValueError(f'Unsupported version for Multi-Turn: {self.version}') From b3298f5cd9b0aacd77ca8c544f1fbbb20b69a1e7 Mon Sep 17 00:00:00 2001 From: hanchaow Date: Fri, 1 Aug 2025 09:47:17 +0800 Subject: [PATCH 2/8] Update __init__.py Add import of QtuneVL series Python code. --- vlmeval/vlm/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vlmeval/vlm/__init__.py b/vlmeval/vlm/__init__.py index 3061e9f9d..d91640fea 100644 --- a/vlmeval/vlm/__init__.py +++ b/vlmeval/vlm/__init__.py @@ -103,3 +103,7 @@ from .treevgr import TreeVGR from .glm4_1v import GLM4_1v from .varco_vision import VarcoVision +from .qtunevl import ( + QTuneVL, + QTuneVLChat, +) From 4447c47ee2b2a1857f5bfd1fc1358d4db716edf9 Mon Sep 17 00:00:00 2001 From: hanchaow Date: Fri, 1 Aug 2025 09:53:22 +0800 Subject: [PATCH 3/8] Update config.py Add QtuneVL series config --- vlmeval/config.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/vlmeval/config.py b/vlmeval/config.py index aa1b7a317..f59ced13e 100644 --- a/vlmeval/config.py +++ b/vlmeval/config.py @@ -1515,6 +1515,22 @@ ), } +# QTuneVL series +qtunevl_series = { + "QTuneVL1.5-2B": partial( + QTuneVLChat, model_path="hanchaow/QTuneVL1.5-2B", version="V1.5" + ), + + "QTuneVL1-3B": partial( + QTuneVL, + model_path="hanchaow/QTuneVL1.5-3B", + min_pixels=1280 * 28 * 28, + max_pixels=16384 * 28 * 28, + use_custom_prompt=True, + post_process=True + ), +} + internvl_groups = [ internvl, internvl2, internvl2_5, mini_internvl, internvl2_5_mpo, internvl3, @@ -1537,7 +1553,7 @@ aria_series, smolvlm_series, sail_series, valley_series, vita_series, ross_series, emu_series, ola_series, ursa_series, gemma_series, long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, - flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series + flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series ] for grp in model_groups: From 831e6ff86c84ec0eaf46c81497864309221552c8 Mon Sep 17 00:00:00 2001 From: hanchaow Date: Fri, 1 Aug 2025 10:51:07 +0800 Subject: [PATCH 4/8] Update qtune_vl_chat.py add function auto_split_flag --- vlmeval/vlm/qtunevl/qtune_vl_chat.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vlmeval/vlm/qtunevl/qtune_vl_chat.py b/vlmeval/vlm/qtunevl/qtune_vl_chat.py index 2e9ed0d4c..9a8fc6fb0 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl_chat.py +++ b/vlmeval/vlm/qtunevl/qtune_vl_chat.py @@ -20,6 +20,21 @@ IMAGENET_MEAN = (0.485, 0.456, 0.406) IMAGENET_STD = (0.229, 0.224, 0.225) +def auto_split_flag(): + flag = os.environ.get('AUTO_SPLIT', '0') + if flag == '1': + return True + _, world_size = get_rank_and_world_size() + try: + import torch + device_count = torch.cuda.device_count() + if device_count > world_size and device_count % world_size == 0: + return True + else: + return False + except: + return False + def build_transform(input_size): MEAN, STD = IMAGENET_MEAN, IMAGENET_STD From 075cb96cf5a8da44510157ced36287310413c3ac Mon Sep 17 00:00:00 2001 From: hanchaow Date: Mon, 4 Aug 2025 13:09:11 +0800 Subject: [PATCH 5/8] Update qtune_vl.py solve ImportError: cannot import name 'Qwen2_5_VLForConditionalGeneration' from 'transformers' --- vlmeval/vlm/qtunevl/qtune_vl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlmeval/vlm/qtunevl/qtune_vl.py b/vlmeval/vlm/qtunevl/qtune_vl.py index bdc301e93..aa8f794e9 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl.py +++ b/vlmeval/vlm/qtunevl/qtune_vl.py @@ -8,7 +8,6 @@ from ..base import BaseModel from ...smp import listinstr -from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor class QTuneVLPromptMixin: @@ -201,6 +200,7 @@ def __init__( self.model_path = model_path self.processor = AutoProcessor.from_pretrained(model_path) + from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_path, torch_dtype='auto', device_map="auto", attn_implementation='flash_attention_2' ) From 230ebb0d6316ea0fdade4046c9f439f3af64a415 Mon Sep 17 00:00:00 2001 From: kennymckormick Date: Mon, 4 Aug 2025 14:03:22 +0800 Subject: [PATCH 6/8] [Fix] Fix Lint --- vlmeval/vlm/qtunevl/__init__.py | 2 +- vlmeval/vlm/qtunevl/qtune_vl.py | 9 ++++----- vlmeval/vlm/qtunevl/qtune_vl_chat.py | 23 ++++++++++------------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/vlmeval/vlm/qtunevl/__init__.py b/vlmeval/vlm/qtunevl/__init__.py index b53f2a4ce..56c06c5dd 100644 --- a/vlmeval/vlm/qtunevl/__init__.py +++ b/vlmeval/vlm/qtunevl/__init__.py @@ -1,4 +1,4 @@ from .qtune_vl import QTuneVL from .qtune_vl_chat import QTuneVLChat -__all__ = ['QTuneVL', 'QTuneVLChat'], +__all__ = ['QTuneVL', 'QTuneVLChat'], diff --git a/vlmeval/vlm/qtunevl/qtune_vl.py b/vlmeval/vlm/qtunevl/qtune_vl.py index aa8f794e9..2fed51c3c 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl.py +++ b/vlmeval/vlm/qtunevl/qtune_vl.py @@ -130,7 +130,7 @@ def _build_vqa_prompt(self, line, dataset: str) -> list[dict[str, str]]: assert msgs[-1]['type'] == 'text' msgs[-1]['value'] += VQA_PROMPT return msgs - + def ensure_image_url(image: str) -> str: prefixes = ['http://', 'https://', 'file://', 'data:image;'] @@ -172,6 +172,7 @@ def __init__( verbose: bool = False, **kwargs, ): + from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor super().__init__(use_custom_prompt=use_custom_prompt) self.min_pixels = min_pixels self.max_pixels = max_pixels @@ -199,8 +200,7 @@ def __init__( assert model_path is not None self.model_path = model_path self.processor = AutoProcessor.from_pretrained(model_path) - - from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor + self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_path, torch_dtype='auto', device_map="auto", attn_implementation='flash_attention_2' ) @@ -276,7 +276,7 @@ def generate_inner_transformers(self, message, dataset=None): print(f'\033[31m{messages}\033[0m') text = self.processor.apply_chat_template([messages], tokenize=False, add_generation_prompt=True) - + images, videos = process_vision_info([messages]) inputs = self.processor(text=text, images=images, videos=videos, padding=True, return_tensors='pt') # noqa: E501 inputs = inputs.to('cuda') @@ -316,4 +316,3 @@ def generate_inner_transformers(self, message, dataset=None): def generate_inner(self, message, dataset=None): return self.generate_inner_transformers(message, dataset=dataset) - diff --git a/vlmeval/vlm/qtunevl/qtune_vl_chat.py b/vlmeval/vlm/qtunevl/qtune_vl_chat.py index 9a8fc6fb0..fcb0e0aa3 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl_chat.py +++ b/vlmeval/vlm/qtunevl/qtune_vl_chat.py @@ -20,6 +20,7 @@ IMAGENET_MEAN = (0.485, 0.456, 0.406) IMAGENET_STD = (0.229, 0.224, 0.225) + def auto_split_flag(): flag = os.environ.get('AUTO_SPLIT', '0') if flag == '1': @@ -62,6 +63,7 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_ best_ratio = ratio return best_ratio + ###################################################################################### def preprocess_padding_global(image, net_input_size=448): # 不形变的resize方式 @@ -74,7 +76,7 @@ def preprocess_padding_global(image, net_input_size=448): scale_ratio = net_input_size / orig_width scale_h = int(orig_height * scale_ratio) scale_w = net_input_size - + resized = image.resize((scale_w, scale_h), Image.BILINEAR) new_image = Image.new('RGB', (net_input_size, net_input_size), (0, 0, 0)) paste_x = (net_input_size - scale_w) // 2 @@ -82,6 +84,7 @@ def preprocess_padding_global(image, net_input_size=448): new_image.paste(resized, (paste_x, paste_y)) return new_image + def dynamic_preprocess(image, min_num=5, max_num=6, image_size=448, use_thumbnail=False): orig_width, orig_height = image.size aspect_ratio = orig_width / orig_height @@ -195,7 +198,7 @@ def load_image2(image_file, input_size=448, target_aspect_ratio=(1, 1), min_num= pixel_values = [transform(image) for image in images] pixel_values = torch.stack(pixel_values) return pixel_values - + def dynamic_preprocess3(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False): orig_width, orig_height = image.size @@ -661,8 +664,7 @@ def generate_v2(self, message, dataset=None): num_patches_list, pixel_values_list = [], [] for image_idx, file_name in enumerate(image_path): - if dataset is not None and (listinstr(['MMBench'], dataset) or - listinstr(['MMStar'], dataset) or listinstr(['MMVet'], dataset)): + if dataset is not None and listinstr(['MMBench', 'MMStar', 'MMVet'], dataset): upscale_flag = image_idx == 0 and dataset is not None and listinstr(['MMMU'], dataset) curr_pixel_values = load_image3( file_name, max_num=max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) @@ -676,8 +678,7 @@ def generate_v2(self, message, dataset=None): curr_pixel_values2 = load_image2( file_name, target_aspect_ratio=target_aspect_ratio, max_num=max_num) curr_pixel_values2 = curr_pixel_values2.cuda().to(torch.bfloat16) - if (listinstr(['MathVista'], dataset) or - listinstr(['HallusionBench'], dataset) or listinstr(['OCRBench'], dataset)): + if listinstr(['MathVista', 'HallusionBench', 'OCRBench'], dataset): curr_pixel_values = torch.cat( (curr_pixel_values[:-1], curr_pixel_values2[:-1], curr_pixel_values[-1:]), 0) else: @@ -687,15 +688,12 @@ def generate_v2(self, message, dataset=None): pixel_values_list.append(curr_pixel_values) pixel_values = torch.cat(pixel_values_list, dim=0) elif image_num == 1: - - - if dataset is not None and (listinstr(['MMBench'], dataset) or - listinstr(['MMStar'], dataset) or listinstr(['MMVet'], dataset)): + if dataset is not None and listinstr(['MMBench', 'MMStar', 'MMVet'], dataset): image_path = [x['value'] for x in message if x['type'] == 'image'][0] upscale_flag = dataset is not None and listinstr(['MMMU'], dataset) pixel_values = load_image3( image_path, max_num=max_num, upscale=upscale_flag).to(self.device).to(torch.bfloat16) - num_patches_list = [pixel_values.size(0)] + num_patches_list = [pixel_values.size(0)] else: image_path = [x['value'] for x in message if x['type'] == 'image'][0] upscale_flag = dataset is not None and listinstr(['MMMU'], dataset) @@ -705,8 +703,7 @@ def generate_v2(self, message, dataset=None): pixel_values2 = load_image2( image_path, target_aspect_ratio=target_aspect_ratio, max_num=max_num) pixel_values2 = pixel_values2.cuda().to(torch.bfloat16) - if (listinstr(['MathVista'], dataset) or - listinstr(['HallusionBench'], dataset) or listinstr(['OCRBench'], dataset)): + if listinstr(['MathVista', 'HallusionBench', 'OCRBench'], dataset): pixel_values = torch.cat((pixel_values[:-1], pixel_values2[:-1], pixel_values[-1:]), 0) else: pixel_values = torch.cat((pixel_values2[:-1], pixel_values[-1:]), 0) From 8d34698b7719b707b54e1bb3b5f9d1a3a6b64b6d Mon Sep 17 00:00:00 2001 From: hanchaow Date: Wed, 6 Aug 2025 10:55:08 +0800 Subject: [PATCH 7/8] Update config.py modify the model's name --- vlmeval/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vlmeval/config.py b/vlmeval/config.py index f59ced13e..e1fad8e21 100644 --- a/vlmeval/config.py +++ b/vlmeval/config.py @@ -1517,13 +1517,13 @@ # QTuneVL series qtunevl_series = { - "QTuneVL1.5-2B": partial( - QTuneVLChat, model_path="hanchaow/QTuneVL1.5-2B", version="V1.5" + "QTuneVL1_5-2B": partial( + QTuneVLChat, model_path="hanchaow/QTuneVL1_5-2B", version="V1.5" ), - "QTuneVL1-3B": partial( + "QTuneVL1_5-3B": partial( QTuneVL, - model_path="hanchaow/QTuneVL1.5-3B", + model_path="hanchaow/QTuneVL1_5-3B", min_pixels=1280 * 28 * 28, max_pixels=16384 * 28 * 28, use_custom_prompt=True, From bbde15f7083690d4aaadbdac436896131a7ecb11 Mon Sep 17 00:00:00 2001 From: FangXinyu-0913 Date: Wed, 6 Aug 2025 17:00:05 +0800 Subject: [PATCH 8/8] fix lint and move mode to cuda --- README.md | 2 +- vlmeval/vlm/qtunevl/qtune_vl.py | 2 +- vlmeval/vlm/qtunevl/qtune_vl_chat.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9dfcd55fb..36a2d93ec 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ English | [简体中文](/docs/zh-CN/README_zh-CN.md) | [日本語](/docs/ja/REA **VLMEvalKit** (the python package name is **vlmeval**) is an **open-source evaluation toolkit** of **large vision-language models (LVLMs)**. It enables **one-command evaluation** of LVLMs on various benchmarks, without the heavy workload of data preparation under multiple repositories. In VLMEvalKit, we adopt **generation-based evaluation** for all LVLMs, and provide the evaluation results obtained with both **exact matching** and **LLM-based answer extraction**. ## Recent Codebase Changes -- **[2025-08-04]** In [**PR 1175**](https://github.com/open-compass/VLMEvalKit/pull/1175), we refine the `can_infer_option` and `can_infer_text`, which increasingly route the evaluation to LLM choice extractors and empirically leads to slight performance improvement for MCQ benchmarks. +- **[2025-08-04]** In [**PR 1175**](https://github.com/open-compass/VLMEvalKit/pull/1175), we refine the `can_infer_option` and `can_infer_text`, which increasingly route the evaluation to LLM choice extractors and empirically leads to slight performance improvement for MCQ benchmarks. ## 🆕 News - **[2025-07-07]** Supported [**SeePhys**](https://seephys.github.io/), which is a ​full spectrum multimodal benchmark for evaluating physics reasoning across different knowledge levels. thanks to [**Quinn777**](https://github.com/Quinn777) 🔥🔥🔥 diff --git a/vlmeval/vlm/qtunevl/qtune_vl.py b/vlmeval/vlm/qtunevl/qtune_vl.py index 2fed51c3c..e634e0283 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl.py +++ b/vlmeval/vlm/qtunevl/qtune_vl.py @@ -204,7 +204,7 @@ def __init__( self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_path, torch_dtype='auto', device_map="auto", attn_implementation='flash_attention_2' ) - self.model.eval() + self.model.eval().to('cuda') torch.cuda.empty_cache() diff --git a/vlmeval/vlm/qtunevl/qtune_vl_chat.py b/vlmeval/vlm/qtunevl/qtune_vl_chat.py index fcb0e0aa3..538cdf5d5 100644 --- a/vlmeval/vlm/qtunevl/qtune_vl_chat.py +++ b/vlmeval/vlm/qtunevl/qtune_vl_chat.py @@ -527,7 +527,7 @@ def __init__(self, torch_dtype=torch.bfloat16, load_in_8bit=load_in_8bit, trust_remote_code=True, - low_cpu_mem_usage=True).eval().cuda() + low_cpu_mem_usage=True).eval().to('cuda') self.device = 'cuda' if best_of_n > 1: @@ -711,7 +711,8 @@ def generate_v2(self, message, dataset=None): else: pixel_values = None num_patches_list = [] - + self.model.vision_model.to(self.device) + self.model.to(self.device) response_list = [] for idx in range(self.best_of_n): kwargs_default = self.kwargs.copy()