From 121ac87c684d4845ae85146dadf0860590920af2 Mon Sep 17 00:00:00 2001 From: Zhoujianqian <15205085056@163.com> Date: Fri, 12 Jun 2026 10:02:03 +0800 Subject: [PATCH 1/2] vllm: add MiniCPM-V 4.6 support (MiniCPMV4_6ForConditionalGeneration) Signed-off-by: Zhoujianqian <15205085056@163.com> --- vllm/patches/vllm_for_multi_arc.patch | 1947 ++++++++++++++++++++++--- 1 file changed, 1726 insertions(+), 221 deletions(-) diff --git a/vllm/patches/vllm_for_multi_arc.patch b/vllm/patches/vllm_for_multi_arc.patch index 9e305f1b..e230ec9e 100644 --- a/vllm/patches/vllm_for_multi_arc.patch +++ b/vllm/patches/vllm_for_multi_arc.patch @@ -10001,10 +10001,18 @@ index 3434716b8..0b601b4b8 100644 super().__init__() diff --git a/vllm/model_executor/models/idefics2_vision_model.py b/vllm/model_executor/models/idefics2_vision_model.py -index c78ad6479..ce1bddd27 100644 +index c78ad6479..865937d6c 100644 --- a/vllm/model_executor/models/idefics2_vision_model.py +++ b/vllm/model_executor/models/idefics2_vision_model.py -@@ -30,6 +30,7 @@ from transformers.models.idefics2.configuration_idefics2 import ( +@@ -22,6 +22,7 @@ from collections.abc import Iterable + + import torch + from torch import nn ++from torch.nn import functional as F + from transformers.models.idefics2.configuration_idefics2 import ( + Idefics2Config, + Idefics2VisionConfig, +@@ -30,6 +31,7 @@ from transformers.models.idefics2.configuration_idefics2 import ( from vllm.distributed import get_tensor_model_parallel_world_size from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention @@ -10012,7 +10020,7 @@ index c78ad6479..ce1bddd27 100644 from vllm.model_executor.layers.conv import Conv2dLayer from vllm.model_executor.layers.linear import ( ColumnParallelLinear, -@@ -161,9 +162,12 @@ class Idefics2VisionAttention(nn.Module): +@@ -161,22 +163,65 @@ class Idefics2VisionAttention(nn.Module): prefix=f"{prefix}.out_proj", disable_tp=use_data_parallel, ) @@ -10027,23 +10035,93 @@ index c78ad6479..ce1bddd27 100644 ) def forward( -@@ -176,7 +180,15 @@ class Idefics2VisionAttention(nn.Module): + self, + hidden_states: torch.Tensor, ++ attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj( + hidden_states + ) # batch_size, q_len, 3 * num_heads_per_partition * head_dim query_states, key_states, value_states = qkv.chunk(3, dim=-1) - # Use unified MMEncoderAttention implementation +- # Use unified MMEncoderAttention implementation - out = self.attn(query_states, key_states, value_states) -+ # add cu_seqlens and max_seqlen for variable-length attention support -+ bsz, q_len, _ = query_states.size() -+ tmp = [0] -+ for i in range(bsz): -+ tmp.append(q_len) -+ seqlen = torch.tensor(tmp) -+ cu_seqlens = torch.cumsum(seqlen, dim=0).to(device=query_states.device) -+ max_seqlen = q_len -+ out = self.attn(query_states, key_states, value_states, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) ++ if attention_mask is None: ++ # Use unified MMEncoderAttention implementation ++ # add cu_seqlens and max_seqlen for variable-length attention support ++ bsz, q_len, _ = query_states.size() ++ tmp = [0] ++ for i in range(bsz): ++ tmp.append(q_len) ++ seqlen = torch.tensor(tmp) ++ cu_seqlens = torch.cumsum(seqlen, dim=0).to(device=query_states.device) ++ max_seqlen = q_len ++ out = self.attn( ++ query_states, ++ key_states, ++ value_states, ++ cu_seqlens=cu_seqlens, ++ max_seqlen=max_seqlen, ++ ) ++ else: ++ # When an explicit (additive) attention mask is supplied (e.g. ++ # MiniCPM-V NaViT-style packed batches), fall back to Torch SDPA so ++ # the mask is honored and cross-image attention is prevented. ++ bsz, q_len = query_states.size()[:2] ++ kv_len = key_states.size(1) ++ query = query_states.view( ++ bsz, q_len, self.num_heads_per_partition, self.head_dim ++ ).transpose(1, 2) ++ key = key_states.view( ++ bsz, kv_len, self.num_heads_per_partition, self.head_dim ++ ).transpose(1, 2) ++ value = value_states.view( ++ bsz, kv_len, self.num_heads_per_partition, self.head_dim ++ ).transpose(1, 2) ++ out = F.scaled_dot_product_attention( ++ query, ++ key, ++ value, ++ attn_mask=attention_mask, ++ dropout_p=0.0, ++ scale=self.scale, ++ ) ++ out = out.transpose(1, 2).reshape(bsz, q_len, -1) attn_output, _ = self.out_proj(out) return attn_output +@@ -244,6 +289,7 @@ class Idefics2EncoderLayer(nn.Module): + def forward( + self, + hidden_states: torch.Tensor, ++ attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """ + Args: +@@ -253,7 +299,7 @@ class Idefics2EncoderLayer(nn.Module): + """ + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) +- hidden_states = self.self_attn(hidden_states) ++ hidden_states = self.self_attn(hidden_states, attention_mask=attention_mask) + hidden_states += residual + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) +@@ -332,12 +378,14 @@ class Idefics2VisionTransformer(nn.Module): + require_post_norm: bool = True, + prefix: str = "", + use_data_parallel: bool = False, ++ apply_encoder_attention_mask: bool = False, + ) -> None: + super().__init__() + + embed_dim = config.hidden_size + self.config = config + self.use_data_parallel = use_data_parallel ++ self.apply_encoder_attention_mask = apply_encoder_attention_mask + self.embeddings = Idefics2VisionEmbeddings(config) + self.encoder = Idefics2Encoder( + config, diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index a8a476abb..b346c2148 100644 --- a/vllm/model_executor/models/interfaces.py @@ -10193,214 +10271,1589 @@ index d149c3642..f3ec5b759 100644 + ]: + return MambaStateCopyFuncCalculator.kda_state_copy_func() + - def compute_logits( - self, - hidden_states: torch.Tensor, -diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py -index 142ad3d6d..b4d5ae415 100644 ---- a/vllm/model_executor/models/lfm2.py -+++ b/vllm/model_executor/models/lfm2.py -@@ -20,6 +20,8 @@ from vllm.model_executor.layers.linear import ( - ) - from vllm.model_executor.layers.logits_processor import LogitsProcessor - from vllm.model_executor.layers.mamba.mamba_utils import ( -+ MambaStateCopyFunc, -+ MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - ) -@@ -455,14 +457,19 @@ class Lfm2ForCausalLM( - conv_kernel=hf_config.conv_L_cache, - ) - -+ @classmethod -+ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc]: -+ return MambaStateCopyFuncCalculator.short_conv_state_copy_func() + def compute_logits( + self, + hidden_states: torch.Tensor, +diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py +index 142ad3d6d..b4d5ae415 100644 +--- a/vllm/model_executor/models/lfm2.py ++++ b/vllm/model_executor/models/lfm2.py +@@ -20,6 +20,8 @@ from vllm.model_executor.layers.linear import ( + ) + from vllm.model_executor.layers.logits_processor import LogitsProcessor + from vllm.model_executor.layers.mamba.mamba_utils import ( ++ MambaStateCopyFunc, ++ MambaStateCopyFuncCalculator, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + ) +@@ -455,14 +457,19 @@ class Lfm2ForCausalLM( + conv_kernel=hf_config.conv_L_cache, + ) + ++ @classmethod ++ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc]: ++ return MambaStateCopyFuncCalculator.short_conv_state_copy_func() ++ + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + cache_config = vllm_config.cache_config +- +- assert not cache_config.enable_prefix_caching, ( +- "Lfm2 currently does not support prefix caching" +- ) ++ if cache_config.mamba_cache_mode == "all": ++ raise NotImplementedError( ++ "Lfm2 currently does not support 'all' prefix caching, " ++ "please use '--mamba-cache-mode=align' instead" ++ ) + + super().__init__() + self.config = config +diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py +index 6677eb9f9..fbac52acf 100644 +--- a/vllm/model_executor/models/lfm2_moe.py ++++ b/vllm/model_executor/models/lfm2_moe.py +@@ -25,6 +25,8 @@ from vllm.model_executor.layers.linear import ( + ) + from vllm.model_executor.layers.logits_processor import LogitsProcessor + from vllm.model_executor.layers.mamba.mamba_utils import ( ++ MambaStateCopyFunc, ++ MambaStateCopyFuncCalculator, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + ) +@@ -636,6 +638,10 @@ class Lfm2MoeForCausalLM( + conv_kernel=hf_config.conv_L_cache, + ) + ++ @classmethod ++ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc]: ++ return MambaStateCopyFuncCalculator.short_conv_state_copy_func() ++ + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config +diff --git a/vllm/model_executor/models/mamba.py b/vllm/model_executor/models/mamba.py +index aa16640a9..85212feca 100644 +--- a/vllm/model_executor/models/mamba.py ++++ b/vllm/model_executor/models/mamba.py +@@ -16,6 +16,8 @@ from vllm.model_executor.layers.layernorm import RMSNorm + from vllm.model_executor.layers.logits_processor import LogitsProcessor + from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer + from vllm.model_executor.layers.mamba.mamba_utils import ( ++ MambaStateCopyFunc, ++ MambaStateCopyFuncCalculator, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + ) +@@ -261,6 +263,10 @@ class MambaForCausalLM( + conv_kernel=hf_config.conv_kernel, + ) + ++ @classmethod ++ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: ++ return MambaStateCopyFuncCalculator.mamba1_state_copy_func() ++ + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): + return self.mamba_cache.copy_inputs_before_cuda_graphs(input_buffers, **kwargs) + +diff --git a/vllm/model_executor/models/mamba2.py b/vllm/model_executor/models/mamba2.py +index 5fcfa9431..ed363df21 100644 +--- a/vllm/model_executor/models/mamba2.py ++++ b/vllm/model_executor/models/mamba2.py +@@ -15,6 +15,8 @@ from vllm.model_executor.layers.layernorm import RMSNorm + from vllm.model_executor.layers.logits_processor import LogitsProcessor + from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 + from vllm.model_executor.layers.mamba.mamba_utils import ( ++ MambaStateCopyFunc, ++ MambaStateCopyFuncCalculator, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + ) +@@ -228,6 +230,10 @@ class Mamba2ForCausalLM( + conv_kernel=hf_config.conv_kernel, + ) + ++ @classmethod ++ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: ++ return MambaStateCopyFuncCalculator.mamba2_state_copy_func() ++ + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + config = vllm_config.model_config.hf_config + +diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py +index 930ff737b..f1a2bd356 100644 +--- a/vllm/model_executor/models/minicpmv.py ++++ b/vllm/model_executor/models/minicpmv.py +@@ -562,7 +562,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + mm_limits = {"image": None} +- if self.get_model_version() in {(2, 6), (4, 0), (4, 5)}: ++ if self.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: + mm_limits["video"] = None + + return mm_limits +@@ -840,7 +840,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): + out_keys: set[str], + ) -> dict[str, NestedTensors]: + # This processor supports zipping prompt and mm_data together +- if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}: ++ if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: + inputs = super()._call_hf_processor( + prompt=prompts, # type: ignore + mm_data=mm_data, +@@ -1047,9 +1047,17 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): + quant_config=quant_config, + prefix=maybe_prefix(prefix, "resampler"), + ) ++ self._resampler_moved = False + + self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors + ++ def _ensure_resampler_device(self) -> None: ++ if self._resampler_moved: ++ return ++ # Only move device, DO NOT touch dtype (fp8 quant needs its own dtype) ++ self.resampler.to(current_platform.device_type) ++ self._resampler_moved = True ++ + def _parse_and_validate_vision_input( + self, + modality: str, +@@ -1171,7 +1179,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) +- return loader.load_weights(weights) ++ loaded = loader.load_weights(weights) ++ self._ensure_resampler_device() ++ return loaded + + def get_mm_mapping(self) -> MultiModelKeys: + """ +@@ -1391,7 +1401,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA): + patch_attention_mask=patch_attn_mask.unsqueeze(1), + tgt_sizes=None, + ) +- + return self.resampler(vision_embedding, tgt_sizes) + + +@@ -1485,12 +1494,13 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA): + patch_attention_mask=patch_attn_mask.unsqueeze(1), + tgt_sizes=tgt_sizes, + ) +- + return self.resampler(vision_embedding, tgt_sizes) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) +- return loader.load_weights(weights) ++ loaded = loader.load_weights(weights) ++ self._ensure_resampler_device() ++ return loaded + + + class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): +@@ -1550,10 +1560,7 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): + quant_config=quant_config, + prefix=prefix, + ) +- +- return resampler.to( +- device=current_platform.device_type, dtype=torch.get_default_dtype() +- ) ++ return resampler.to(dtype=torch.get_default_dtype()) + + def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: + pixel_values = data["pixel_values"] +@@ -1588,7 +1595,9 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) +- return loader.load_weights(weights) ++ loaded = loader.load_weights(weights) ++ self._ensure_resampler_device() ++ return loaded + + + class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): +@@ -1685,12 +1694,13 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): + patch_attention_mask=patch_attn_mask.unsqueeze(1), + tgt_sizes=tgt_sizes, + ) +- + return self.resampler(vision_embedding, tgt_sizes, all_temporal_ids) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) +- return loader.load_weights(weights) ++ loaded = loader.load_weights(weights) ++ self._ensure_resampler_device() ++ return loaded + + + _SUPPORT_VERSION = { +diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py +new file mode 100644 +index 000000000..713b38426 +--- /dev/null ++++ b/vllm/model_executor/models/minicpmv4_6.py +@@ -0,0 +1,1351 @@ ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++"""Inference-only MiniCPM-V 4.6 model (MiniCPMV4_6ForConditionalGeneration).""" ++ ++from collections.abc import Iterable, Mapping ++from typing import Any ++ ++import numpy as np ++import torch ++from PIL import Image as PILImage ++from torch import nn ++from transformers import MiniCPMV4_6Config ++ ++from vllm.config import VllmConfig ++from vllm.distributed import get_tensor_model_parallel_world_size ++from vllm.model_executor.layers.activation import get_act_fn ++from vllm.model_executor.layers.attention.mm_encoder_attention import ( ++ MMEncoderAttention, ++) ++from vllm.model_executor.layers.linear import ( ++ QKVParallelLinear, ++ RowParallelLinear, ++) ++from vllm.model_executor.layers.mamba.mamba_utils import ( ++ MambaStateCopyFuncCalculator, ++ MambaStateDtypeCalculator, ++ MambaStateShapeCalculator, ++) ++from vllm.model_executor.layers.quantization import QuantizationConfig ++from vllm.model_executor.model_loader.weight_utils import default_weight_loader ++from vllm.multimodal import MULTIMODAL_REGISTRY ++from vllm.multimodal.inputs import ( ++ MultiModalFeatureSpec, ++ MultiModalFieldConfig, ++ NestedTensors, ++) ++from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessorItems ++from vllm.multimodal.processing import ( ++ PromptReplacement, ++ PromptUpdateDetails, ++ _seq2text, ++) ++from vllm.sequence import IntermediateTensors ++ ++from .idefics2_vision_model import Idefics2VisionTransformer ++from .interfaces import ( ++ HasInnerState, ++ IsHybrid, ++ MultiModalEmbeddings, ++ SupportsMRoPE, ++ SupportsMultiModal, ++ SupportsPP, ++ _require_is_multimodal, ++) ++from .minicpmv import ( ++ MiniCPMVDummyInputsBuilder, ++ MiniCPMVImageEmbeddingInputs, ++ MiniCPMVImageEmbeddingItems, ++ MiniCPMVImagePixelInputs, ++ MiniCPMVMultiModalProcessor, ++ MiniCPMVProcessingInfo, ++ MiniCPMVVideoEmbeddingItems, ++) ++from .module_mapping import MultiModelKeys ++from .qwen3_5 import Qwen3_5ForCausalLM ++from .utils import ( ++ AutoWeightsLoader, ++ WeightsMapper, ++ _merge_multimodal_embeddings, ++ flatten_bn, ++ maybe_prefix, ++) ++from .vision import is_vit_use_data_parallel ++ ++ ++def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]): ++ fields = dict( ++ pixel_values=MultiModalFieldConfig.batched("image"), ++ tgt_sizes=MultiModalFieldConfig.batched("image"), ++ image_embeds=MultiModalFieldConfig.batched("image"), ++ video_pixel_values=MultiModalFieldConfig.batched("video"), ++ video_image_sizes=MultiModalFieldConfig.batched("video"), ++ video_tgt_sizes=MultiModalFieldConfig.batched("video"), ++ video_embeds=MultiModalFieldConfig.batched("video"), ++ ) ++ if "use_vit_merger" in hf_inputs: ++ fields["use_vit_merger"] = MultiModalFieldConfig.batched("image") ++ return fields ++ ++ ++class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): ++ def _recompute_cached_prompt_update(self, cached_update, new_item_idx): ++ # transformers v5.7+ moved the im-id tokens from the image processor ++ # onto the tokenizer, so the base class branch that reads ++ # ``image_processor.im_id_start`` no longer works for 4.6. Rebuild the ++ # ``{idx}`` prefix from tokenizer attributes and ++ # use the proper embedding selector token. ++ new_update = super( ++ MiniCPMVMultiModalProcessor, self ++ )._recompute_cached_prompt_update(cached_update, new_item_idx) ++ ++ if cached_update.modality in ("image", "video"): ++ tokenizer = self.info.get_tokenizer() ++ id_start = getattr(tokenizer, "image_id_start_token", "") ++ id_end = getattr(tokenizer, "image_id_end_token", "") ++ if cached_update.modality == "image": ++ embed_text = getattr(tokenizer, "image_token", "<|image_pad|>") ++ else: ++ embed_text = getattr(tokenizer, "video_token", "<|video_pad|>") ++ ++ text = _seq2text(tokenizer, cached_update.content.full) ++ prev_item_idx = cached_update.item_idx ++ new_update = new_update.with_content( ++ PromptUpdateDetails.select_text( ++ text.replace( ++ f"{id_start}{prev_item_idx}{id_end}", ++ f"{id_start}{new_item_idx}{id_end}", ++ 1, ++ ), ++ embed_text, ++ ) ++ ) ++ ++ return new_update ++ ++ def _resolve_downsample_mode( ++ self, ++ mm_kwargs: Mapping[str, object], ++ ) -> str: ++ ds = mm_kwargs.get("downsample_mode") ++ if ds is not None: ++ return str(ds) ++ return self.info._get_downsample_mode() ++ ++ def get_image_prompt_texts( ++ self, ++ image_size, ++ image_idx: int = 0, ++ downsample_mode: str | None = None, ++ ) -> str: ++ return self.info.get_slice_image_placeholder( ++ image_size, ++ image_idx=image_idx, ++ downsample_mode=downsample_mode, ++ ) ++ ++ def get_video_prompt_texts( ++ self, ++ image_size, ++ num_frames: int, ++ downsample_mode: str | None = None, ++ video_idx: int = 0, ++ ) -> str: ++ # Match transformers v5.7+ MiniCPMV4_6Processor video formatting: ++ # {video_idx}(VIDEO*src ++ # VIDEO*patch...)*num_frames ++ # Crucially the visual token inside each frame is ``<|video_pad|>`` ++ # (tokenizer.video_token), NOT ``<|image_pad|>`` — they share the same ++ # embedding-injection role but the language model is conditioned on ++ # which one is used. Using image_token for video silently produces ++ # garbage descriptions. ++ info = self.info ++ grids, source_tokens, patch_tokens = info._compute_visual_tokens( ++ image_size, ++ max_slice_nums=info.get_video_max_slice_num(), ++ downsample_mode=downsample_mode, ++ ) ++ tokenizer = info.get_tokenizer() ++ video_token = getattr(tokenizer, "video_token", "<|video_pad|>") ++ image_start = getattr(tokenizer, "image_start_token", "") ++ image_end = getattr(tokenizer, "image_end_token", "") ++ slice_start = getattr(tokenizer, "slice_start_token", "") ++ slice_end = getattr(tokenizer, "slice_end_token", "") ++ id_start = getattr(tokenizer, "image_id_start_token", "") ++ id_end = getattr(tokenizer, "image_id_end_token", "") ++ ++ per_frame = image_start + video_token * source_tokens + image_end ++ if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0: ++ slice_ph = slice_start + video_token * patch_tokens + slice_end ++ rows = [slice_ph * grids[0] for _ in range(grids[1])] ++ per_frame += "\n".join(rows) ++ ++ body = per_frame * num_frames ++ return f"{id_start}{video_idx}{id_end}" + body ++ ++ def process_images( ++ self, ++ mm_data: Mapping[str, object], ++ mm_kwargs: Mapping[str, object], ++ tok_kwargs: Mapping[str, object], ++ ) -> Mapping[str, NestedTensors]: ++ if (images := mm_data.get("images")) is None: ++ return {} ++ ++ mm_items = self._get_data_parser().parse_mm_data({"image": images}) ++ parsed_images = mm_items.get_items( ++ "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems) ++ ) ++ ++ if isinstance(parsed_images, MiniCPMVImageEmbeddingItems): ++ return {} ++ ++ # transformers v5.7+ MiniCPMV4_6ImageProcessor returns ++ # `pixel_values` (1, C, P, sum_W) where all slices are fused along W ++ # (NaViT-style), and `target_sizes` (n_slices, 2). vLLM expects each ++ # image entry to be a 4D tensor (n_slices, C, P, L_max_padded). ++ n_images = len(parsed_images) ++ image_processor = self.info.get_image_processor() ++ patch_size = image_processor.patch_size ++ per_image_pixel_values: list[torch.Tensor] = [] ++ per_image_tgt_sizes: list[torch.Tensor] = [] ++ for image in parsed_images: ++ ip_out = image_processor([image], **mm_kwargs) ++ pv = ip_out["pixel_values"] # (1, C, P, sum_W) ++ ts = ip_out["target_sizes"] # (n_slices, 2) ++ if pv.ndim == 4 and pv.shape[0] == 1: ++ pv = pv.squeeze(0) # (C, P, sum_W) ++ ts_long = ts.to(torch.long) ++ split_widths = (ts_long[:, 0] * ts_long[:, 1] * patch_size).tolist() ++ slices = torch.split(pv, split_widths, dim=-1) ++ n_slices = len(slices) ++ l_max = max(s.shape[-1] for s in slices) ++ out = torch.zeros( ++ n_slices, ++ pv.shape[0], ++ pv.shape[1], ++ l_max, ++ dtype=pv.dtype, ++ device=pv.device, ++ ) ++ for i, s in enumerate(slices): ++ out[i, :, :, : s.shape[-1]] = s ++ per_image_pixel_values.append(out) ++ per_image_tgt_sizes.append(ts_long) ++ ++ image_inputs: dict = { ++ "pixel_values": per_image_pixel_values, ++ "tgt_sizes": per_image_tgt_sizes, ++ } ++ ++ ds_mode = self._resolve_downsample_mode(mm_kwargs) ++ insert_layer_id = getattr( ++ self.info.get_hf_config(), ++ "insert_layer_id", ++ -1, ++ ) ++ merger_flag = ds_mode != "4x" and insert_layer_id >= 0 ++ image_inputs["use_vit_merger"] = [ ++ torch.tensor([merger_flag], dtype=torch.bool) for _ in range(n_images) ++ ] ++ return image_inputs ++ ++ def process_videos( ++ self, ++ mm_data: Mapping[str, object], ++ mm_kwargs: Mapping[str, object], ++ tok_kwargs: Mapping[str, object], ++ ) -> Mapping[str, NestedTensors]: ++ if (videos := mm_data.get("videos")) is None: ++ return {} ++ ++ mm_items = self._get_data_parser().parse_mm_data({"video": videos}) ++ parsed_videos = mm_items.get_items( ++ "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems) ++ ) ++ ++ if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems): ++ return {} ++ ++ # Treat each video as a sequence of frames. The transformers v5.7+ ++ # `MiniCPMV4_6ImageProcessor` returns NaViT-style fused `pixel_values`; ++ # we run it per-frame, split the slices, then re-pack each video into ++ # a single 4D tensor (sum_slices, C, P, L_max_video). ++ image_processor = self.info.get_image_processor() ++ patch_size = image_processor.patch_size ++ video_max_slice = self.info.get_video_max_slice_num() ++ video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice} ++ ++ per_video_pixel_values: list[torch.Tensor] = [] ++ per_video_tgt_sizes: list[torch.Tensor] = [] ++ per_video_image_sizes: list[torch.Tensor] = [] ++ ++ for video in parsed_videos: ++ # video is iterable of frames (PIL Image or numpy array). ++ all_slices: list[torch.Tensor] = [] ++ ts_list: list[torch.Tensor] = [] ++ frame_sizes: list[torch.Tensor] = [] ++ for frame in video: ++ # Record per-frame (W, H) for video_image_sizes so that ++ # get_video_prompt_texts can consume a consistent frame size. ++ if isinstance(frame, PILImage.Image): ++ w, h = frame.size ++ elif isinstance(frame, np.ndarray): ++ if frame.ndim == 3 and frame.shape[-1] in (1, 3, 4): ++ # HWC (e.g. from np.array(PIL.Image)) ++ h, w = frame.shape[0], frame.shape[1] ++ else: ++ # CHW ++ _, h, w = frame.shape ++ elif isinstance(frame, torch.Tensor): ++ if frame.ndim == 3 and frame.shape[-1] in (1, 3, 4): ++ h, w = frame.shape[0], frame.shape[1] ++ else: ++ _, h, w = frame.shape ++ else: ++ raise TypeError(f"Unsupported frame type: {type(frame)}") ++ frame_sizes.append(torch.tensor([w, h], dtype=torch.long, device="cpu")) ++ ++ ip_out = image_processor([frame], **video_mm_kwargs) ++ pv = ip_out["pixel_values"] # (1, C, P, sum_W) ++ ts = ip_out["target_sizes"] # (n_slices, 2) ++ if pv.ndim == 4 and pv.shape[0] == 1: ++ pv = pv.squeeze(0) # (C, P, sum_W) ++ ts_long = ts.to(torch.long) ++ split_widths = (ts_long[:, 0] * ts_long[:, 1] * patch_size).tolist() ++ slices = torch.split(pv, split_widths, dim=-1) ++ all_slices.extend(slices) ++ ts_list.append(ts_long) ++ ++ if not all_slices: ++ continue ++ ++ l_max = max(s.shape[-1] for s in all_slices) ++ n_total = len(all_slices) ++ C, P = all_slices[0].shape[0], all_slices[0].shape[1] ++ out = torch.zeros( ++ n_total, ++ C, ++ P, ++ l_max, ++ dtype=all_slices[0].dtype, ++ device=all_slices[0].device, ++ ) ++ for i, s in enumerate(all_slices): ++ out[i, :, :, : s.shape[-1]] = s ++ ++ per_video_pixel_values.append(out) ++ per_video_tgt_sizes.append(torch.cat(ts_list, dim=0)) ++ per_video_image_sizes.append(torch.stack(frame_sizes)) ++ ++ if not per_video_pixel_values: ++ return {} ++ ++ return { ++ "video_pixel_values": per_video_pixel_values, ++ "video_tgt_sizes": per_video_tgt_sizes, ++ "video_image_sizes": per_video_image_sizes, ++ } ++ ++ def _get_prompt_updates( ++ self, ++ mm_items, ++ hf_processor_mm_kwargs: Mapping[str, object], ++ out_mm_kwargs, ++ ): ++ ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs) ++ ++ placeholders = [ ++ ("image", self.info.image_pattern), ++ ("video", self.info.video_pattern), ++ ] ++ tokenizer = self.info.get_tokenizer() ++ additional_placeholders = [] ++ for modality, pattern in placeholders: ++ sub_pattern = tokenizer.decode( ++ tokenizer.encode(pattern, add_special_tokens=False) ++ ) ++ if sub_pattern != pattern: ++ additional_placeholders.append((modality, sub_pattern)) ++ placeholders += additional_placeholders ++ ++ # The 4.6 chat_template emits `<|image_pad|>` / `<|video_pad|>` rather ++ # than ``, so use those tokens as the embedding selector. ++ image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>") ++ video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>") ++ ++ def get_image_replacement(item_idx: int): ++ images = mm_items.get_items( ++ "image", ++ (MiniCPMVImageEmbeddingItems, ImageProcessorItems), ++ ) ++ image_size = images.get_image_size(item_idx) ++ return PromptUpdateDetails.select_text( ++ self.get_image_prompt_texts( ++ image_size, ++ item_idx, ++ downsample_mode=ds_mode, ++ ), ++ image_embed_text, ++ ) ++ ++ def get_video_replacement(item_idx: int): ++ # Prefer video_image_sizes from processed data so that the ++ # placeholder count is driven by the same frame sizes that the ++ # vision tower will actually consume. ++ video_mm_kwargs = out_mm_kwargs.get("video") ++ if video_mm_kwargs is not None and item_idx < len(video_mm_kwargs): ++ video_item = video_mm_kwargs[item_idx] ++ image_sizes_elem = video_item.get("video_image_sizes") ++ if image_sizes_elem is not None and image_sizes_elem.data is not None: ++ # image_sizes_elem.data: (num_frames, 2) – each row is [W, H] ++ image_sizes = image_sizes_elem.data ++ num_frames = image_sizes.shape[0] ++ frame_size = ImageSize( ++ width=int(image_sizes[0, 0].item()), ++ height=int(image_sizes[0, 1].item()), ++ ) ++ return PromptUpdateDetails.select_text( ++ self.get_video_prompt_texts( ++ frame_size, ++ num_frames, ++ downsample_mode=ds_mode, ++ video_idx=item_idx, ++ ), ++ video_embed_text, ++ ) ++ ++ videos = mm_items.get_items( ++ "video", ++ (MiniCPMVVideoEmbeddingItems, VideoProcessorItems), ++ ) ++ frame_size = videos.get_frame_size(item_idx) ++ num_frames = videos.get_num_frames(item_idx) ++ return PromptUpdateDetails.select_text( ++ self.get_video_prompt_texts( ++ frame_size, ++ num_frames, ++ downsample_mode=ds_mode, ++ video_idx=item_idx, ++ ), ++ video_embed_text, ++ ) ++ ++ get_replacement = { ++ "image": get_image_replacement, ++ "video": get_video_replacement, ++ } ++ ++ return [ ++ PromptReplacement( ++ modality=modality, ++ target=pattern, ++ replacement=get_replacement[modality], ++ ) ++ for modality, pattern in placeholders ++ ] ++ ++ def _get_mm_fields_config( ++ self, ++ hf_inputs, ++ hf_processor_mm_kwargs: Mapping[str, object], ++ ) -> Mapping[str, MultiModalFieldConfig]: ++ return _minicpmv4_6_field_config(hf_inputs) ++ ++ ++class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo): ++ # transformers v5.7+ chat_template emits these as image/video placeholders. ++ image_pattern = "<|image_pad|>" ++ video_pattern = "<|video_pad|>" ++ ++ def get_hf_config(self): ++ return self.ctx.get_hf_config() ++ ++ def get_hf_processor(self, **kwargs: object): ++ # MiniCPM-V 4.6 keeps the native transformers MiniCPMV4_6Processor: ++ # this model has its own image/video handling and prompt-update logic ++ # below, so it does not need (and is incompatible with) the vendored ++ # MiniCPMVProcessor used by 2.x/4.0/4.5, whose __init__ assumes a ++ # legacy `image_processor.version` attribute that 4.6 no longer has. ++ hf_processor = self.ctx.get_hf_processor(**kwargs) ++ ++ # NumPy arrays are considered as Iterable but not Sequence in ++ # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428 ++ image_processor = getattr(hf_processor, "image_processor", None) ++ if image_processor is not None: ++ # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std` ++ for attr in ("mean", "std", "image_mean", "image_std"): ++ val = getattr(image_processor, attr, None) ++ if isinstance(val, np.ndarray): ++ setattr(image_processor, attr, val.tolist()) ++ ++ return hf_processor ++ ++ def _get_expected_hidden_size(self) -> int: ++ config = self.get_hf_config() ++ if hasattr(config, "text_config") and config.text_config is not None: ++ return config.text_config.hidden_size ++ return config.hidden_size ++ ++ def get_model_version(self): ++ return (4, 6) ++ ++ def get_supported_mm_limits(self) -> Mapping[str, int | None]: ++ return {"image": None, "video": None} ++ ++ def get_image_max_slice_num(self) -> int: ++ config = self.get_hf_config() ++ if hasattr(config, "slice_config") and config.slice_config is not None: ++ return getattr(config.slice_config, "max_slice_nums", 9) ++ return getattr(config, "max_slice_nums", 9) ++ ++ def get_video_max_slice_num(self) -> int: ++ # Override the base class default of 1: transformers v5.7+ ++ # `MiniCPMV4_6VideoProcessor` keeps the same max_slice_nums (default 9) ++ # as the image processor so that high-res frames get sliced. ++ try: ++ hf_processor = self.get_hf_processor() ++ video_processor = getattr(hf_processor, "video_processor", None) ++ if video_processor is not None: ++ return int(getattr(video_processor, "max_slice_nums", 9)) ++ except Exception: ++ pass ++ return self.get_image_max_slice_num() ++ ++ def _get_downsample_mode( ++ self, ++ downsample_mode: str | None = None, ++ ) -> str: ++ if downsample_mode is not None: ++ return downsample_mode ++ image_processor = self.get_image_processor() ++ return getattr(image_processor, "downsample_mode", "16x") ++ ++ def _compute_visual_tokens( ++ self, ++ image_size, ++ max_slice_nums: int | None = None, ++ downsample_mode: str | None = None, ++ ) -> tuple[list[int], int, int]: ++ """Compute grid, source_image_visual_tokens and patch_visual_tokens. ++ ++ Args: ++ downsample_mode: ``"16x"`` (default, full merge) or ``"4x"`` ++ (skip vit_merger, 4x more visual tokens). ++ ++ Returns: ++ (grids, source_image_visual_tokens, patch_visual_tokens) ++ grids is [0, 0] when no slicing occurs. ++ """ ++ image_processor = self.get_image_processor() ++ if max_slice_nums is None: ++ max_slice_nums = image_processor.max_slice_nums ++ ++ patch_size = image_processor.patch_size ++ scale_res = image_processor.scale_resolution ++ downsample_mode = self._get_downsample_mode(downsample_mode) ++ token_divisor = 4 if downsample_mode == "4x" else 16 ++ ++ # transformers v5.7+ requires `scale_resolution` arg ++ try: ++ grids = image_processor.get_sliced_grid( ++ image_size, ++ max_slice_nums, ++ scale_res, ++ ) ++ except TypeError: ++ grids = image_processor.get_sliced_grid( ++ image_size, ++ max_slice_nums, ++ ) ++ ++ if grids is None: ++ best_size = image_processor.find_best_resize( ++ image_size, ++ scale_res, ++ patch_size, ++ allow_upscale=True, ++ ) ++ source_tokens = ( ++ best_size[0] * best_size[1] // (patch_size * patch_size * token_divisor) ++ ) ++ return [0, 0], source_tokens, 0 ++ ++ best_resize = image_processor.find_best_resize( ++ image_size, ++ scale_res, ++ patch_size, ++ ) ++ source_tokens = ( ++ best_resize[0] * best_resize[1] // (patch_size * patch_size * token_divisor) ++ ) ++ refine_size = image_processor.get_refine_size( ++ image_size, ++ grids, ++ scale_res, ++ patch_size, ++ allow_upscale=True, ++ ) ++ patch_w = refine_size[0] // grids[0] ++ patch_h = refine_size[1] // grids[1] ++ patch_tokens = patch_w * patch_h // (patch_size * patch_size * token_divisor) ++ return grids, source_tokens, patch_tokens ++ ++ def get_slice_image_placeholder( ++ self, ++ image_size, ++ image_idx: int = 0, ++ max_slice_nums: int | None = None, ++ use_image_id: bool = True, ++ downsample_mode: str | None = None, ++ ) -> str: ++ grids, source_tokens, patch_tokens = self._compute_visual_tokens( ++ image_size, ++ max_slice_nums, ++ downsample_mode=downsample_mode, ++ ) ++ image_processor = self.get_image_processor() ++ # transformers v5.7+ removed `get_slice_image_placeholder` from the ++ # image_processor and moved the logic into MiniCPMV4_6Processor. ++ # Replicate it here using tokenizer special tokens. ++ if hasattr(image_processor, "get_slice_image_placeholder"): ++ return image_processor.get_slice_image_placeholder( ++ grids, ++ image_idx=image_idx, ++ max_slice_nums=max_slice_nums, ++ use_image_id=use_image_id, ++ source_image_visual_tokens=source_tokens, ++ patch_visual_tokens=patch_tokens, ++ ) ++ tokenizer = self.get_tokenizer() ++ image_token = getattr(tokenizer, "image_token", "<|image_pad|>") ++ image_start = getattr(tokenizer, "image_start_token", "") ++ image_end = getattr(tokenizer, "image_end_token", "") ++ slice_start = getattr(tokenizer, "slice_start_token", "") ++ slice_end = getattr(tokenizer, "slice_end_token", "") ++ id_start = getattr(tokenizer, "image_id_start_token", "") ++ id_end = getattr(tokenizer, "image_id_end_token", "") ++ ++ placeholder = image_start + image_token * source_tokens + image_end ++ if use_image_id: ++ placeholder = f"{id_start}{image_idx}{id_end}" + placeholder ++ ++ num_cols, num_rows = grids[0], grids[1] ++ if num_cols > 0 and num_rows > 0 and patch_tokens > 0: ++ slice_ph = slice_start + image_token * patch_tokens + slice_end ++ slices = [slice_ph * num_cols for _ in range(num_rows)] ++ placeholder += "\n".join(slices) ++ return placeholder ++ ++ def get_num_image_tokens( ++ self, ++ image_size, ++ max_slice_nums: int | None = None, ++ downsample_mode: str | None = None, ++ ) -> int: ++ grids, source_tokens, patch_tokens = self._compute_visual_tokens( ++ image_size, ++ max_slice_nums, ++ downsample_mode=downsample_mode, ++ ) ++ return source_tokens + grids[0] * grids[1] * patch_tokens ++ ++ ++class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module): ++ def __init__( ++ self, ++ config, ++ quant_config: QuantizationConfig | None = None, ++ prefix: str = "", ++ ): ++ super().__init__() ++ use_data_parallel = is_vit_use_data_parallel() ++ self.embed_dim = config.hidden_size ++ self.num_heads = config.num_attention_heads ++ self.head_dim = self.embed_dim // self.num_heads ++ self.scale = self.head_dim**-0.5 ++ ++ tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size() ++ assert self.num_heads % tp_size == 0 ++ self.num_heads_per_partition = self.num_heads // tp_size ++ ++ self.qkv_proj = QKVParallelLinear( ++ self.embed_dim, ++ self.head_dim, ++ self.num_heads, ++ quant_config=quant_config, ++ prefix=f"{prefix}.qkv_proj", ++ disable_tp=use_data_parallel, ++ ) ++ self.out_proj = RowParallelLinear( ++ self.embed_dim, ++ self.embed_dim, ++ bias=True, ++ quant_config=quant_config, ++ prefix=f"{prefix}.out_proj", ++ disable_tp=use_data_parallel, ++ ) ++ self.attn = MMEncoderAttention( ++ self.num_heads_per_partition, ++ self.head_dim, ++ self.scale, ++ prefix=f"{prefix}.attn", ++ ) ++ ++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ++ qkv, _ = self.qkv_proj(hidden_states) ++ q, k, v = qkv.chunk(3, dim=-1) ++ attn_out = self.attn(q, k, v) ++ out, _ = self.out_proj(attn_out) ++ return out ++ ++ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: ++ stacked_params_mapping = [ ++ ("qkv_proj", "q_proj", "q"), ++ ("qkv_proj", "k_proj", "k"), ++ ("qkv_proj", "v_proj", "v"), ++ ] ++ params_dict = dict(self.named_parameters()) ++ loaded_params: set[str] = set() ++ for name, loaded_weight in weights: ++ for param_name, weight_name, shard_id in stacked_params_mapping: ++ if weight_name not in name: ++ continue ++ mapped_name = name.replace(weight_name, param_name, 1) ++ if mapped_name not in params_dict: ++ continue ++ param = params_dict[mapped_name] ++ param.weight_loader(param, loaded_weight, shard_id) ++ # Record the *fused* param name so the loader's ++ # "uninitialized weights" check sees qkv_proj as loaded. ++ name = mapped_name ++ break ++ else: ++ if name not in params_dict: ++ continue ++ param = params_dict[name] ++ weight_loader = getattr(param, "weight_loader", default_weight_loader) ++ weight_loader(param, loaded_weight) ++ loaded_params.add(name) ++ return loaded_params ++ ++ ++class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module): ++ def __init__( ++ self, ++ config, ++ quant_config: QuantizationConfig | None = None, ++ prefix: str = "", ++ ): ++ super().__init__() ++ self.window_kernel_size = (2, 2) ++ self.embed_dim = config.hidden_size ++ ++ self.self_attn = MiniCPMV4_6ViTWindowAttentionSelfAttn( ++ config, ++ quant_config=quant_config, ++ prefix=f"{prefix}.self_attn", ++ ) ++ self.layer_norm1 = nn.LayerNorm( ++ self.embed_dim, ++ eps=config.layer_norm_eps, ++ ) ++ ++ hidden_4x = self.embed_dim * 4 ++ inter_4x = config.intermediate_size * 4 ++ ++ self.pre_norm = nn.LayerNorm(hidden_4x, eps=config.layer_norm_eps) ++ self.linear_1 = nn.Linear(hidden_4x, inter_4x, bias=True) ++ self.act = get_act_fn("gelu_pytorch_tanh") ++ self.linear_2 = nn.Linear(inter_4x, self.embed_dim, bias=True) ++ ++ def _apply_window_attention( ++ self, ++ valid_states: torch.Tensor, ++ H: int, ++ W: int, ++ ) -> torch.Tensor: ++ D = valid_states.shape[-1] ++ wh, ww = self.window_kernel_size ++ nh, nw = H // wh, W // ww ++ num_windows = nh * nw ++ ++ x = valid_states.view(H, W, D) ++ x = x.view(nh, wh, nw, ww, D).permute(0, 2, 1, 3, 4).contiguous() ++ x = x.view(num_windows, wh * ww, D) ++ ++ x = self.self_attn(x) ++ ++ x = x.view(nh, nw, wh, ww, D).permute(0, 2, 1, 3, 4).contiguous() ++ return x.view(H * W, D) ++ ++ def _apply_mlp_downsample( ++ self, ++ valid_states: torch.Tensor, ++ H: int, ++ W: int, ++ ) -> torch.Tensor: ++ D = valid_states.shape[-1] ++ wh, ww = self.window_kernel_size ++ nh, nw = H // wh, W // ww ++ ++ x = valid_states.view(H, W, D) ++ x = x.view(nh, wh, nw, ww, D).permute(0, 2, 1, 3, 4).contiguous() ++ ++ residual = x.reshape(nh * nw, wh * ww, D).mean(dim=1) ++ x = x.reshape(nh * nw, wh * ww * D) ++ ++ x = self.pre_norm(x) ++ x = self.linear_1(x) ++ x = self.act(x) ++ x = self.linear_2(x) ++ return x + residual ++ ++ def forward( ++ self, ++ hidden_states: torch.Tensor, ++ tgt_sizes: torch.Tensor, ++ attention_mask: torch.Tensor | None, ++ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: ++ B, _L, D = hidden_states.shape ++ device = hidden_states.device ++ dtype = hidden_states.dtype ++ ++ all_merged = [] ++ new_tgt_sizes = torch.zeros_like(tgt_sizes) ++ ++ for b in range(B): ++ H, W = tgt_sizes[b].tolist() ++ hs = hidden_states[b, : H * W, :] ++ ++ residual = hs ++ hs = self.layer_norm1(hs) ++ hs = residual + self._apply_window_attention(hs, H, W) ++ ++ wh, ww = self.window_kernel_size ++ new_H, new_W = H // wh, W // ww ++ all_merged.append(self._apply_mlp_downsample(hs, H, W)) ++ new_tgt_sizes[b] = torch.tensor( ++ [new_H, new_W], ++ device=device, ++ dtype=tgt_sizes.dtype, ++ ) ++ ++ new_num_patches = new_tgt_sizes[:, 0] * new_tgt_sizes[:, 1] ++ new_max_patches = int(new_num_patches.max().item()) ++ new_hidden = torch.zeros( ++ B, ++ new_max_patches, ++ D, ++ device=device, ++ dtype=dtype, ++ ) ++ for b, merged in enumerate(all_merged): ++ new_hidden[b, : merged.shape[0], :] = merged ++ ++ # Build new attention mask after spatial downsampling ++ new_attention_mask: torch.Tensor | None = None ++ if attention_mask is not None: ++ mask = torch.zeros( ++ B, ++ new_max_patches, ++ dtype=torch.bool, ++ device=device, ++ ) ++ for b in range(B): ++ mask[b, : int(new_num_patches[b].item())] = True ++ min_val = torch.finfo(dtype).min ++ new_attention_mask = (~mask).to(dtype=dtype) * min_val ++ new_attention_mask = new_attention_mask[:, None, None, :] ++ ++ return new_hidden, new_tgt_sizes, new_attention_mask ++ ++ ++class MiniCPMV4_6DownsampleMLP(nn.Module): ++ """Match HF (transformers v5.7+) parameter naming: pre_norm/linear_1/ ++ act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2)).""" ++ ++ def __init__( ++ self, ++ hidden_size: int, ++ llm_embed_dim: int, ++ merge_kernel_size: tuple[int, int] = (2, 2), ++ ): ++ super().__init__() ++ self.merge_kernel_size = merge_kernel_size ++ self.hidden_size = hidden_size * merge_kernel_size[0] * merge_kernel_size[1] ++ self.pre_norm = nn.LayerNorm(self.hidden_size, eps=1e-6) ++ self.linear_1 = nn.Linear(self.hidden_size, self.hidden_size, bias=True) ++ self.act = get_act_fn("gelu") ++ self.linear_2 = nn.Linear(self.hidden_size, llm_embed_dim, bias=True) ++ ++ def forward(self, x: torch.Tensor) -> torch.Tensor: ++ x = self.pre_norm(x) ++ x = self.linear_1(x) ++ x = self.act(x) ++ x = self.linear_2(x) ++ return x ++ ++ ++class MiniCPMV4_6Merger(nn.Module): ++ def __init__( ++ self, ++ hidden_size: int, ++ llm_embed_dim: int, ++ merge_kernel_size: tuple[int, int] = (2, 2), ++ times: int = 1, ++ ): ++ super().__init__() ++ self.merge_kernel_size = merge_kernel_size ++ self.times = times ++ self.mlp = nn.ModuleList( ++ [ ++ MiniCPMV4_6DownsampleMLP( ++ hidden_size, ++ llm_embed_dim if i == times - 1 else hidden_size, ++ merge_kernel_size, ++ ) ++ for i in range(times) ++ ] ++ ) ++ ++ def forward( ++ self, ++ hidden_states: torch.Tensor, ++ tgt_sizes: torch.Tensor, ++ ) -> list[torch.Tensor]: ++ """ ++ Args: ++ hidden_states: (B, max_patches, D) padded batch. ++ tgt_sizes: (B, 2) actual (H, W) per sample. ++ """ ++ m1, m2 = self.merge_kernel_size ++ results = [] ++ ++ for b in range(len(tgt_sizes)): ++ h, w = tgt_sizes[b].tolist() ++ n_patches = h * w ++ hs = hidden_states[b, :n_patches, :] ++ ++ hs = hs.reshape(h // m1, m1, w // m2, m2, -1) ++ hs = hs.permute(0, 2, 1, 3, 4).reshape( ++ (h // m1) * (w // m2), ++ m1 * m2 * hs.shape[-1], ++ ) ++ hs = self.mlp[0](hs) ++ ++ if self.times > 1: ++ cur_h, cur_w = h // m1, w // m2 ++ for t in range(1, self.times): ++ cur_h, cur_w = cur_h // m1, cur_w // m2 ++ hs = hs.reshape(cur_h, m1, cur_w, m2, -1) ++ hs = hs.permute(0, 2, 1, 3, 4).reshape( ++ cur_h * cur_w, ++ m1 * m2 * hs.shape[-1], ++ ) ++ hs = self.mlp[t](hs) ++ ++ results.append(hs) ++ ++ return results ++ ++ ++@MULTIMODAL_REGISTRY.register_processor( ++ MiniCPMV4_6MultiModalProcessor, ++ info=MiniCPMV4_6ProcessingInfo, ++ dummy_inputs=MiniCPMVDummyInputsBuilder, ++) ++class MiniCPMV4_6ForConditionalGeneration( ++ nn.Module, ++ SupportsMultiModal, ++ SupportsPP, ++ HasInnerState, ++ IsHybrid, ++ SupportsMRoPE, ++): ++ supports_encoder_tp_data = True ++ ++ hf_to_vllm_mapper = WeightsMapper( ++ orig_to_new_prefix={ ++ # transformers v5.7+ uses `vision_tower` and nests `vit_merger` ++ # inside it. Order matters: more specific prefix must come first. ++ "model.vision_tower.vit_merger.": "vit_merger.", ++ "model.vision_tower.": "vpm.", ++ "model.vpm.": "vpm.", ++ "model.vit_merger.": "vit_merger.", ++ "model.merger.": "merger.", ++ "model.language_model.": "language_model.model.", ++ "lm_head.": "language_model.lm_head.", ++ } ++ ) ++ ++ packed_modules_mapping = { ++ "qkv_proj": ["q_proj", "k_proj", "v_proj"], ++ "gate_up_proj": ["gate_proj", "up_proj"], ++ "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], ++ "in_proj_ba": ["in_proj_b", "in_proj_a"], ++ } ++ ++ @classmethod ++ def get_placeholder_str(cls, modality: str, i: int) -> str | None: ++ # transformers v5.7+ chat_template uses these tokens. ++ if modality.startswith("image"): ++ return "<|image_pad|>" ++ if modality.startswith("video"): ++ return "<|video_pad|>" ++ raise ValueError("Only image or video modality is supported") ++ ++ def get_mrope_input_positions( ++ self, ++ input_tokens: list[int], ++ mm_features: list["MultiModalFeatureSpec"], ++ ) -> tuple[torch.Tensor, int]: ++ """MiniCPM-V uses embedding injection for vision, not spatial M-RoPE. ++ ++ All tokens (text and vision placeholders) get identical sequential ++ positions duplicated across the 3 M-RoPE channels expected by the ++ Qwen3.5 backbone. ++ """ ++ seq_len = len(input_tokens) ++ positions = torch.arange(seq_len).unsqueeze(0).expand(3, -1) ++ return positions, 0 ++ ++ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): ++ super().__init__() ++ config: MiniCPMV4_6Config = vllm_config.model_config.hf_config ++ quant_config = vllm_config.quant_config ++ multimodal_config = vllm_config.model_config.multimodal_config ++ ++ self.config = config ++ self.multimodal_config = multimodal_config ++ self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" ++ ++ # --- Vision tower --- ++ # NOTE(intel-xpu): upstream wraps tower/LM construction in ++ # self._mark_tower_model / self._mark_language_model context ++ # managers for encoder data-parallel marking. Those helpers are ++ # not present in this baseline (mirrors the Intel qwen3_5 ++ # adaptation), so the submodules are built directly. ++ self.vpm = Idefics2VisionTransformer( ++ config.vision_config, ++ quant_config=quant_config, ++ apply_encoder_attention_mask=True, ++ prefix=maybe_prefix(prefix, "vpm"), ++ ) ++ if config.drop_vision_last_layer: ++ self.vpm.encoder.layers = self.vpm.encoder.layers[:-1] ++ ++ self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger( ++ config.vision_config, ++ quant_config=quant_config, ++ prefix=maybe_prefix(prefix, "vit_merger"), ++ ) ++ self.merger = MiniCPMV4_6Merger( ++ hidden_size=config.vision_config.hidden_size, ++ llm_embed_dim=config.text_config.hidden_size, ++ ) ++ ++ # --- Language model --- ++ # Temporarily swap top-level model_type so that Qwen3_5ForCausalLM ++ # picks up the expected text config when introspecting the hf config. ++ saved_model_type = config.model_type ++ config.model_type = "qwen3_5_text" ++ try: ++ self.language_model = Qwen3_5ForCausalLM( ++ vllm_config=vllm_config, ++ prefix=maybe_prefix(prefix, "language_model"), ++ ) ++ finally: ++ config.model_type = saved_model_type ++ ++ self.make_empty_intermediate_tensors = ( ++ self.language_model.make_empty_intermediate_tensors ++ ) ++ ++ # ----- Multimodal parsing ----- ++ ++ def _parse_and_validate_vision_input( ++ self, ++ **kwargs: object, ++ ) -> MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs | None: ++ pixel_values = kwargs.pop("pixel_values", None) ++ image_embeds = kwargs.pop("image_embeds", None) ++ ++ if pixel_values is None and image_embeds is None: ++ return None ++ ++ if image_embeds is not None: ++ return MiniCPMVImageEmbeddingInputs( ++ type="image_embeds", ++ image_embeds=image_embeds, ++ ) ++ ++ tgt_sizes = kwargs.pop("tgt_sizes") ++ num_slices_flat = torch.tensor([len(ps) for ps in pixel_values]) ++ pixel_values_flat = flatten_bn(pixel_values) ++ tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True) ++ ++ return MiniCPMVImagePixelInputs( ++ type="pixel_values", ++ pixel_values=pixel_values_flat, ++ tgt_sizes=tgt_sizes_flat, ++ num_slices=num_slices_flat, ++ ) ++ ++ # ----- Vision forward ----- ++ ++ def get_vision_hidden_states( ++ self, ++ data: MiniCPMVImagePixelInputs, ++ downsample_mode: str | None = None, ++ ) -> list[torch.Tensor]: ++ pixel_values = data["pixel_values"] ++ tgt_sizes = data["tgt_sizes"] ++ ++ B = len(pixel_values) ++ P = pixel_values[0].shape[-2] ++ L = max(item.shape[-1] for item in pixel_values) ++ device = pixel_values[0].device ++ target_dtype = self.vpm.embeddings.patch_embedding.weight.dtype ++ ++ all_pixel_values = torch.zeros( ++ B, ++ 3, ++ P, ++ L, ++ dtype=target_dtype, ++ device=device, ++ ) ++ for i, pv in enumerate(pixel_values): ++ all_pixel_values[i, ..., : pv.shape[-1]] = pv.to(target_dtype) ++ ++ num_patches = tgt_sizes.prod(-1) ++ max_patches = int(num_patches.max().item()) ++ patch_attn_mask = torch.zeros( ++ B, ++ max_patches, ++ dtype=torch.bool, ++ device=device, ++ ) ++ for i in range(B): ++ patch_attn_mask[i, : num_patches[i]] = True ++ ++ hidden_states = self.vpm.embeddings( ++ all_pixel_values, ++ patch_attention_mask=patch_attn_mask.unsqueeze(1), ++ tgt_sizes=tgt_sizes, ++ ) ++ ++ if torch.any(~patch_attn_mask): ++ mask_dtype = hidden_states.dtype ++ min_val = torch.finfo(mask_dtype).min ++ attention_mask = (~patch_attn_mask).to(dtype=mask_dtype) * min_val ++ attention_mask = attention_mask[:, None, None, :] ++ else: ++ attention_mask = None ++ ++ # Encoder layers with mid-encoder merger injection ++ insert_layer_id = getattr(self.config, "insert_layer_id", -1) ++ if downsample_mode is None: ++ downsample_mode = getattr(self.config, "downsample_mode", "16x") ++ use_vit_merger = downsample_mode != "4x" and insert_layer_id >= 0 ++ ++ for layer in self.vpm.encoder.layers[: insert_layer_id + 1]: ++ hidden_states = layer(hidden_states, attention_mask=attention_mask) ++ ++ if use_vit_merger: ++ hidden_states, tgt_sizes, attention_mask = self.vit_merger( ++ hidden_states, ++ tgt_sizes, ++ attention_mask, ++ ) ++ ++ for layer in self.vpm.encoder.layers[insert_layer_id + 1 :]: ++ hidden_states = layer(hidden_states, attention_mask=attention_mask) ++ ++ # 4. Post layernorm ++ hidden_states = self.vpm.post_layernorm(hidden_states) ++ ++ # 5. MLP merger → list of per-slice tensors ++ return self.merger(hidden_states, tgt_sizes) ++ ++ def _process_vision_input(self, image_input, use_vit_merger=None): ++ if image_input["type"] == "image_embeds": ++ return image_input["image_embeds"] ++ ++ downsample_mode = None ++ if use_vit_merger is not None: ++ downsample_mode = "16x" if use_vit_merger else "4x" ++ image_features = self.get_vision_hidden_states( ++ image_input, ++ downsample_mode=downsample_mode, ++ ) ++ num_slices = image_input["num_slices"] ++ results = [] ++ idx = 0 ++ for n in num_slices.tolist(): ++ group = image_features[idx : idx + n] ++ results.append(torch.cat(group, dim=0)) ++ idx += n ++ return results ++ ++ # ----- Multimodal embedding interface ----- ++ ++ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: ++ use_vit_merger_tensors = kwargs.pop("use_vit_merger", None) ++ use_vit_merger = None ++ if use_vit_merger_tensors is not None: ++ if isinstance(use_vit_merger_tensors, torch.Tensor): ++ use_vit_merger = bool(use_vit_merger_tensors.any().item()) ++ elif isinstance(use_vit_merger_tensors, list | tuple): ++ use_vit_merger = any( ++ bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t) ++ for t in use_vit_merger_tensors ++ ) ++ ++ # Split kwargs into image / video buckets (videos are processed via ++ # the same vision pipeline; their fields just carry a ``video_`` prefix). ++ image_kwargs = { ++ k: v ++ for k, v in kwargs.items() ++ if k in ("pixel_values", "image_embeds", "tgt_sizes") ++ } ++ video_kwargs = { ++ k.removeprefix("video_"): v ++ for k, v in kwargs.items() ++ if k.startswith("video_") ++ } ++ ++ multimodal_embeddings: tuple[torch.Tensor, ...] = () ++ ++ if ( ++ image_kwargs.get("pixel_values") is not None ++ or image_kwargs.get("image_embeds") is not None ++ ): ++ image_input = self._parse_and_validate_vision_input(**image_kwargs) ++ if image_input is not None: ++ multimodal_embeddings += tuple( ++ self._process_vision_input( ++ image_input, ++ use_vit_merger=use_vit_merger, ++ ) ++ ) ++ ++ if ( ++ video_kwargs.get("pixel_values") is not None ++ or video_kwargs.get("image_embeds") is not None ++ ): ++ video_input = self._parse_and_validate_vision_input(**video_kwargs) ++ if video_input is not None: ++ multimodal_embeddings += tuple( ++ self._process_vision_input( ++ video_input, ++ use_vit_merger=use_vit_merger, ++ ) ++ ) ++ ++ if not multimodal_embeddings: ++ return [] ++ return multimodal_embeddings ++ ++ def embed_input_ids( ++ self, ++ input_ids: torch.Tensor, ++ multimodal_embeddings: MultiModalEmbeddings | None = None, ++ *, ++ is_multimodal: torch.Tensor | None = None, ++ ) -> torch.Tensor: ++ inputs_embeds = self._embed_text_input_ids( ++ input_ids, ++ self.language_model.embed_input_ids, ++ is_multimodal=is_multimodal, ++ handle_oov_mm_token=False, ++ ) ++ if multimodal_embeddings is None or len(multimodal_embeddings) == 0: ++ return inputs_embeds ++ ++ is_multimodal = _require_is_multimodal(is_multimodal) ++ return _merge_multimodal_embeddings( ++ inputs_embeds=inputs_embeds, ++ multimodal_embeddings=multimodal_embeddings, ++ is_multimodal=is_multimodal, ++ ) ++ ++ # ----- Forward / Logits ----- ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor | None, ++ positions: torch.Tensor, ++ intermediate_tensors: IntermediateTensors | None = None, ++ inputs_embeds: torch.Tensor | None = None, ++ **kwargs: Any, ++ ) -> torch.Tensor: ++ if intermediate_tensors is not None: ++ inputs_embeds = None ++ ++ return self.language_model.model( ++ input_ids=input_ids, ++ positions=positions, ++ intermediate_tensors=intermediate_tensors, ++ inputs_embeds=inputs_embeds, ++ ) + - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - cache_config = vllm_config.cache_config -- -- assert not cache_config.enable_prefix_caching, ( -- "Lfm2 currently does not support prefix caching" -- ) -+ if cache_config.mamba_cache_mode == "all": -+ raise NotImplementedError( -+ "Lfm2 currently does not support 'all' prefix caching, " -+ "please use '--mamba-cache-mode=align' instead" -+ ) - - super().__init__() - self.config = config -diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py -index 6677eb9f9..fbac52acf 100644 ---- a/vllm/model_executor/models/lfm2_moe.py -+++ b/vllm/model_executor/models/lfm2_moe.py -@@ -25,6 +25,8 @@ from vllm.model_executor.layers.linear import ( - ) - from vllm.model_executor.layers.logits_processor import LogitsProcessor - from vllm.model_executor.layers.mamba.mamba_utils import ( -+ MambaStateCopyFunc, -+ MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - ) -@@ -636,6 +638,10 @@ class Lfm2MoeForCausalLM( - conv_kernel=hf_config.conv_L_cache, - ) - -+ @classmethod -+ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc]: -+ return MambaStateCopyFuncCalculator.short_conv_state_copy_func() ++ def compute_logits( ++ self, ++ hidden_states: torch.Tensor, ++ ) -> torch.Tensor | None: ++ return self.language_model.compute_logits(hidden_states) + - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config -diff --git a/vllm/model_executor/models/mamba.py b/vllm/model_executor/models/mamba.py -index aa16640a9..85212feca 100644 ---- a/vllm/model_executor/models/mamba.py -+++ b/vllm/model_executor/models/mamba.py -@@ -16,6 +16,8 @@ from vllm.model_executor.layers.layernorm import RMSNorm - from vllm.model_executor.layers.logits_processor import LogitsProcessor - from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer - from vllm.model_executor.layers.mamba.mamba_utils import ( -+ MambaStateCopyFunc, -+ MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - ) -@@ -261,6 +263,10 @@ class MambaForCausalLM( - conv_kernel=hf_config.conv_kernel, - ) - -+ @classmethod -+ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: -+ return MambaStateCopyFuncCalculator.mamba1_state_copy_func() ++ # ----- Weight loading ----- ++ ++ def load_weights( ++ self, ++ weights: Iterable[tuple[str, torch.Tensor]], ++ ) -> set[str]: ++ loader = AutoWeightsLoader(self, skip_prefixes=["mtp."]) ++ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) ++ ++ def get_mm_mapping(self) -> MultiModelKeys: ++ return MultiModelKeys.from_string_field( ++ language_model="language_model", ++ connector=["vit_merger", "merger"], ++ tower_model="vpm", ++ ) ++ ++ # ----- Mamba / Hybrid state helpers (same as Qwen3.5 VLM) ----- + - def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs): - return self.mamba_cache.copy_inputs_before_cuda_graphs(input_buffers, **kwargs) - -diff --git a/vllm/model_executor/models/mamba2.py b/vllm/model_executor/models/mamba2.py -index 5fcfa9431..ed363df21 100644 ---- a/vllm/model_executor/models/mamba2.py -+++ b/vllm/model_executor/models/mamba2.py -@@ -15,6 +15,8 @@ from vllm.model_executor.layers.layernorm import RMSNorm - from vllm.model_executor.layers.logits_processor import LogitsProcessor - from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 - from vllm.model_executor.layers.mamba.mamba_utils import ( -+ MambaStateCopyFunc, -+ MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - ) -@@ -228,6 +230,10 @@ class Mamba2ForCausalLM( - conv_kernel=hf_config.conv_kernel, - ) - + @classmethod -+ def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: -+ return MambaStateCopyFuncCalculator.mamba2_state_copy_func() ++ def get_mamba_state_dtype_from_config(cls, vllm_config): ++ # NOTE(intel-xpu/b8.3.1): this baseline's ++ # gated_delta_net_state_dtype takes (model_dtype, mamba_cache_dtype); ++ # the upstream 3rd arg (mamba_ssm_cache_dtype) does not exist here. ++ # Aligned with qwen3_5.py in the same baseline. ++ return MambaStateDtypeCalculator.gated_delta_net_state_dtype( ++ vllm_config.model_config.dtype, ++ vllm_config.cache_config.mamba_cache_dtype, ++ ) + - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - config = vllm_config.model_config.hf_config - -diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py -index 930ff737b..5cbcc0300 100644 ---- a/vllm/model_executor/models/minicpmv.py -+++ b/vllm/model_executor/models/minicpmv.py -@@ -1047,9 +1047,17 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): - quant_config=quant_config, - prefix=maybe_prefix(prefix, "resampler"), - ) -+ self._resampler_moved = False - - self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors - -+ def _ensure_resampler_device(self) -> None: -+ if self._resampler_moved: -+ return -+ # Only move device, DO NOT touch dtype (fp8 quant needs its own dtype) -+ self.resampler.to(current_platform.device_type) -+ self._resampler_moved = True ++ @classmethod ++ def get_mamba_state_shape_from_config(cls, vllm_config): ++ parallel_config = vllm_config.parallel_config ++ hf_config = vllm_config.model_config.hf_text_config ++ tp_size = parallel_config.tensor_parallel_size ++ num_spec = ( ++ vllm_config.speculative_config.num_speculative_tokens ++ if vllm_config.speculative_config ++ else 0 ++ ) ++ return MambaStateShapeCalculator.gated_delta_net_state_shape( ++ tp_size, ++ hf_config.linear_num_key_heads, ++ hf_config.linear_num_value_heads, ++ hf_config.linear_key_head_dim, ++ hf_config.linear_value_head_dim, ++ hf_config.linear_conv_kernel_dim, ++ num_spec, ++ ) + - def _parse_and_validate_vision_input( - self, - modality: str, -@@ -1171,7 +1179,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) -- return loader.load_weights(weights) -+ loaded = loader.load_weights(weights) -+ self._ensure_resampler_device() -+ return loaded - - def get_mm_mapping(self) -> MultiModelKeys: - """ -@@ -1391,7 +1401,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA): - patch_attention_mask=patch_attn_mask.unsqueeze(1), - tgt_sizes=None, - ) -- - return self.resampler(vision_embedding, tgt_sizes) - - -@@ -1485,12 +1494,13 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA): - patch_attention_mask=patch_attn_mask.unsqueeze(1), - tgt_sizes=tgt_sizes, - ) -- - return self.resampler(vision_embedding, tgt_sizes) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) -- return loader.load_weights(weights) -+ loaded = loader.load_weights(weights) -+ self._ensure_resampler_device() -+ return loaded - - - class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): -@@ -1550,10 +1560,7 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): - quant_config=quant_config, - prefix=prefix, - ) -- -- return resampler.to( -- device=current_platform.device_type, dtype=torch.get_default_dtype() -- ) -+ return resampler.to(dtype=torch.get_default_dtype()) - - def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: - pixel_values = data["pixel_values"] -@@ -1588,7 +1595,9 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA): - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) -- return loader.load_weights(weights) -+ loaded = loader.load_weights(weights) -+ self._ensure_resampler_device() -+ return loaded - - - class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): -@@ -1685,12 +1694,13 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): - patch_attention_mask=patch_attn_mask.unsqueeze(1), - tgt_sizes=tgt_sizes, - ) -- - return self.resampler(vision_embedding, tgt_sizes, all_temporal_ids) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"]) -- return loader.load_weights(weights) -+ loaded = loader.load_weights(weights) -+ self._ensure_resampler_device() -+ return loaded - - - _SUPPORT_VERSION = { ++ @classmethod ++ def get_mamba_state_copy_func(cls): ++ return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() diff --git a/vllm/model_executor/models/minimax_text_01.py b/vllm/model_executor/models/minimax_text_01.py index 955a73ff1..44417c98b 100644 --- a/vllm/model_executor/models/minimax_text_01.py @@ -15316,7 +16769,7 @@ index 318680448..e89a4441f 100644 if "experts.gate_up_proj" in name or "experts.down_proj" in name: is_fused_expert = True diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py -index 362028ebf..333f5baa1 100644 +index 362028ebf..10f2221c3 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -112,6 +112,7 @@ _TEXT_GENERATION_MODELS = { @@ -15335,7 +16788,18 @@ index 362028ebf..333f5baa1 100644 "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"), "Ernie4_5_VLMoeForConditionalGeneration": ( "ernie45_vl", -@@ -420,11 +422,23 @@ _MULTIMODAL_MODELS = { +@@ -379,6 +381,10 @@ _MULTIMODAL_MODELS = { + ), + "MiniCPMO": ("minicpmo", "MiniCPMO"), + "MiniCPMV": ("minicpmv", "MiniCPMV"), ++ "MiniCPMV4_6ForConditionalGeneration": ( ++ "minicpmv4_6", ++ "MiniCPMV4_6ForConditionalGeneration", ++ ), + "Mistral3ForConditionalGeneration": ( + "mistral3", + "Mistral3ForConditionalGeneration", +@@ -420,11 +426,23 @@ _MULTIMODAL_MODELS = { "qwen3_omni_moe_thinker", "Qwen3OmniMoeThinkerForConditionalGeneration", ), @@ -15359,7 +16823,7 @@ index 362028ebf..333f5baa1 100644 "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"), "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), # noqa: E501 "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501 -@@ -462,9 +476,13 @@ _SPECULATIVE_DECODING_MODELS = { +@@ -462,9 +480,13 @@ _SPECULATIVE_DECODING_MODELS = { "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"), "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"), "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"), @@ -15394,6 +16858,29 @@ index 1bda00653..d70addc8f 100644 ) else: self.attn = attn_cls( +diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py +index 2a4bec774..1c41674b1 100644 +--- a/vllm/model_executor/models/vision.py ++++ b/vllm/model_executor/models/vision.py +@@ -544,3 +544,18 @@ def get_llm_pos_ids_for_vision( + llm_pos_ids_list.append(_llm_pos_ids + start_idx) + llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1) + return llm_pos_ids ++ ++ ++def is_vit_use_data_parallel() -> bool: ++ """Return True when the ViT encoder should run in data-parallel mode. ++ ++ Backported for MiniCPM-V 4.6 support. Reads the active MultiModalConfig ++ from the current vLLM config context. ++ """ ++ from vllm.config.vllm import get_current_vllm_config_or_none ++ ++ vllm_config = get_current_vllm_config_or_none() ++ if vllm_config is None or vllm_config.model_config is None: ++ return False ++ mm_cfg = getattr(vllm_config.model_config, "multimodal_config", None) ++ return mm_cfg is not None and mm_cfg.mm_encoder_tp_mode == "data" diff --git a/vllm/model_executor/models/zamba2.py b/vllm/model_executor/models/zamba2.py index b5132cd86..59a8520f7 100644 --- a/vllm/model_executor/models/zamba2.py @@ -15899,10 +17386,24 @@ index b5088a116..81c096ecb 100644 diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py -index fe84b6c15..0064cc6d6 100644 +index fe84b6c15..ec18514d5 100644 --- a/vllm/transformers_utils/chat_templates/registry.py +++ b/vllm/transformers_utils/chat_templates/registry.py -@@ -34,6 +34,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = { +@@ -21,8 +21,11 @@ def _get_qwen_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None + + + def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None: +- # MiniCPM-V-4.5 version uses a dedicated template +- if "4.5" in tokenizer_name_or_path or "4_5" in tokenizer_name_or_path: ++ # MiniCPM-V-4.5 / 4.6 share the same dedicated template ++ if any( ++ v in tokenizer_name_or_path ++ for v in ("4.5", "4_5", "4.6", "4_6") ++ ): + return CHAT_TEMPLATES_DIR / "template_minicpmv45.jinja" + + # Other versions use chatml template +@@ -34,9 +37,11 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = { "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja", "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja", "deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja", @@ -15910,6 +17411,10 @@ index fe84b6c15..0064cc6d6 100644 "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja", "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja", "minicpmv": _get_minicpmv_chat_template_fallback, ++ "minicpmv4_6": _get_minicpmv_chat_template_fallback, + "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja", + "qwen": _get_qwen_chat_template_fallback, + "siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja", diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 0a7cfffd4..1e302fe75 100644 --- a/vllm/transformers_utils/config.py From e71854d7c300dd73081c27696251eff722174514 Mon Sep 17 00:00:00 2001 From: Zhoujianqian <15205085056@163.com> Date: Mon, 13 Jul 2026 20:41:50 +0800 Subject: [PATCH 2/2] vllm: fix MiniCPM-V-4.6 image (W,H)/(H,W) order in _compute_visual_tokens The XPU MiniCPM-V-4.6 support passed vLLM ImageSize (width, height) directly to transformers image_processor.get_sliced_grid / find_best_resize / get_refine_size, which expect (height, width). For non-square images whose slice grid is asymmetric (e.g. 1084x820) this produced a transposed grid, so the number of image placeholder tokens (495) did not match the vision encoder output (441). The mismatched masked_scatter_ then tripped the XPU indexing assert 'totalElements <= srcSize' (torch-xpu-ops Indexing.cpp), killing EngineCore (EngineDeadError -> HTTP 500). Convert ImageSize to (height, width) before calling the processor, matching upstream vLLM releases/v0.24.0. Verified across 12 image sizes that placeholder count now equals encoder token count, and the original crashing request now succeeds. --- vllm/patches/vllm_for_multi_arc.patch | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/vllm/patches/vllm_for_multi_arc.patch b/vllm/patches/vllm_for_multi_arc.patch index e230ec9e..1adb3e40 100644 --- a/vllm/patches/vllm_for_multi_arc.patch +++ b/vllm/patches/vllm_for_multi_arc.patch @@ -10502,7 +10502,7 @@ new file mode 100644 index 000000000..713b38426 --- /dev/null +++ b/vllm/model_executor/models/minicpmv4_6.py -@@ -0,0 +1,1351 @@ +@@ -0,0 +1,1358 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only MiniCPM-V 4.6 model (MiniCPMV4_6ForConditionalGeneration).""" @@ -11051,22 +11051,29 @@ index 000000000..713b38426 + downsample_mode = self._get_downsample_mode(downsample_mode) + token_divisor = 4 if downsample_mode == "4x" else 16 + ++ # vLLM ImageSize is (width, height); transformers get_sliced_grid/ ++ # find_best_resize/get_refine_size expect (height, width). ++ if hasattr(image_size, "height"): ++ hf_image_size = (image_size.height, image_size.width) ++ else: ++ hf_image_size = (image_size[1], image_size[0]) ++ + # transformers v5.7+ requires `scale_resolution` arg + try: + grids = image_processor.get_sliced_grid( -+ image_size, ++ hf_image_size, + max_slice_nums, + scale_res, + ) + except TypeError: + grids = image_processor.get_sliced_grid( -+ image_size, ++ hf_image_size, + max_slice_nums, + ) + + if grids is None: + best_size = image_processor.find_best_resize( -+ image_size, ++ hf_image_size, + scale_res, + patch_size, + allow_upscale=True, @@ -11077,7 +11084,7 @@ index 000000000..713b38426 + return [0, 0], source_tokens, 0 + + best_resize = image_processor.find_best_resize( -+ image_size, ++ hf_image_size, + scale_res, + patch_size, + ) @@ -11085,7 +11092,7 @@ index 000000000..713b38426 + best_resize[0] * best_resize[1] // (patch_size * patch_size * token_divisor) + ) + refine_size = image_processor.get_refine_size( -+ image_size, ++ hf_image_size, + grids, + scale_res, + patch_size,