code to patch for qwen3.5 grpo training

verl/models/monkey_patch.py

@@ -31,10 +31,12 @@
     "qwen2_5_vl",
     "qwen3_vl",
     "qwen3_vl_moe",
+    "qwen3_5",
 )
 
 QWEN2_VL_MODELS = ("qwen2_vl", "qwen2_5_vl")
 QWEN3_VL_MODELS = ("qwen3_vl", "qwen3_vl_moe")
+QWEN3_5_MODELS = ("qwen3_5",)
 
 
 def apply_ulysses_patch(model_type: str) -> None:
@@ -76,3 +78,10 @@ def apply_ulysses_patch(model_type: str) -> None:
         # TODO: add linear cross entropy kernels
         Qwen3VLForConditionalGeneration.forward = qwen3_vl_model_forward
         Qwen3VLMoeForConditionalGeneration.forward = qwen3_vl_model_forward
+    elif model_type in QWEN3_5_MODELS:
+        # Qwen3.5's default transformers forward handles image embedding, mRoPE position IDs,
+        # and mixed text-image batches correctly. No forward monkey-patching needed.
+        # Position IDs are pre-computed in dataset.py using qwen3_5.get_rope_index.
+        # Image token/feature mismatch is handled by fix_qwen35_image_check.py which
+        # patches the strict check in modeling_qwen3_5.py at build time.
+        pass

verl/models/transformers/flash_attention_utils.py

@@ -21,7 +21,13 @@
 import torch
 import torch.distributed as dist
 from transformers.modeling_flash_attention_utils import _flash_attention_forward, fa_peft_integration_check
-from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10
+from transformers.utils import is_flash_attn_2_available
+
+try:
+    from transformers.utils import is_flash_attn_greater_or_equal_2_10
+except ImportError:
+    from transformers.utils import is_flash_attn_greater_or_equal
+    is_flash_attn_greater_or_equal_2_10 = lambda: is_flash_attn_greater_or_equal("2.10")
 
 from ...utils.ulysses import (
     gather_heads_scatter_seq,

verl/models/transformers/qwen3_5.py

@@ -0,0 +1,146 @@
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+# Adapted from:
+# https://github.com/huggingface/transformers/blob/v5.4.0/src/transformers/models/qwen3_5/modeling_qwen3_5.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Standalone position ID computation for Qwen3.5-VL.
+
+Qwen3.5 uses mRoPE like Qwen2/3-VL but with key differences:
+- Uses mm_token_type_ids (0=text, 1=image, 2=video) instead of scanning for special tokens
+- Position advances by max(H, W) / spatial_merge_size after each vision block (not T*H*W)
+- get_vision_position_ids computes spatial positions with start_position offset
+"""
+
+import itertools
+from typing import Optional
+
+import torch
+from transformers import ProcessorMixin
+
+
+def get_vision_position_ids(
+    start_position: int,
+    grid_thw: torch.Tensor,
+    spatial_merge_size: int = 1,
+    device: Optional[torch.device] = None,
+) -> torch.Tensor:
+    """Compute 3D positional indices for vision tokens from a single image or video.
+
+    Args:
+        start_position: Offset added to all computed positional indices.
+        grid_thw: Tensor of shape (3,) — (T, H, W) grid of the vision feature.
+        spatial_merge_size: Factor by which H and W are reduced in the backbone.
+        device: Device for the output tensor.
+
+    Returns:
+        torch.LongTensor of shape (3, sequence_length): [temporal, height, width] positions.
+    """
+    llm_grid_t = grid_thw[0].item()
+    llm_grid_h = grid_thw[1].item() // spatial_merge_size
+    llm_grid_w = grid_thw[2].item() // spatial_merge_size
+
+    image_seq_length = llm_grid_h * llm_grid_w * llm_grid_t
+    position_width = torch.arange(start_position, start_position + llm_grid_w, device=device).repeat(
+        llm_grid_h * llm_grid_t
+    )
+    position_height = torch.arange(start_position, start_position + llm_grid_h, device=device).repeat_interleave(
+        llm_grid_w * llm_grid_t
+    )
+    position_temporal = torch.full((image_seq_length,), start_position, device=device, dtype=torch.long)
+
+    return torch.stack([position_temporal, position_height, position_width], dim=0)
+
+
+def get_rope_index(
+    processor: "ProcessorMixin",
+    input_ids: torch.Tensor,
+    mm_token_type_ids: torch.Tensor,
+    image_grid_thw: Optional[torch.Tensor] = None,
+    video_grid_thw: Optional[torch.Tensor] = None,
+    attention_mask: Optional[torch.Tensor] = None,
+    **kwargs,
+) -> torch.Tensor:
+    """Compute mRoPE position IDs for Qwen3.5, adapted from Qwen3_5Model.get_rope_index.
+
+    This is a standalone (non-method) version that works on single (unbatched) samples,
+    matching the interface used in EasyR1's dataset.py.
+
+    Args:
+        processor: The Qwen3.5 processor (used to get spatial_merge_size).
+        input_ids: 1D tensor of token IDs (seq_length,).
+        mm_token_type_ids: 1D tensor (seq_length,) — 0=text, 1=image, 2=video.
+        image_grid_thw: Tensor of shape (num_images, 3) or None.
+        video_grid_thw: Tensor of shape (num_videos, 3) or None.
+        attention_mask: 1D tensor (seq_length,) or None.
+
+    Returns:
+        torch.Tensor of shape (3, seq_length): mRoPE position IDs [temporal, height, width].
+    """
+    # Qwen3.5 splits video_grid_thw by temporal dimension (timestamps separate frames)
+    if video_grid_thw is not None:
+        video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0)
+        video_grid_thw[:, 0] = 1
+
+    spatial_merge_size = processor.image_processor.merge_size
+
+    position_ids = torch.zeros(3, len(input_ids), dtype=input_ids.dtype, device=input_ids.device)
+
+    if attention_mask is not None:
+        mask = attention_mask.bool()
+        input_ids_masked = input_ids[mask]
+        mm_types_masked = mm_token_type_ids[mask]
+    else:
+        input_ids_masked = input_ids
+        mm_types_masked = mm_token_type_ids
+
+    grid_iters = {
+        1: iter(image_grid_thw) if image_grid_thw is not None else None,
+        2: iter(video_grid_thw) if video_grid_thw is not None else None,
+    }
+
+    # Group consecutive tokens by modality type
+    input_type_group = []
+    for key, group in itertools.groupby(enumerate(mm_types_masked.tolist()), lambda x: x[1]):
+        group = list(group)
+        start_index = group[0][0]
+        end_index = group[-1][0] + 1
+        input_type_group.append((key, start_index, end_index))
+
+    current_pos = 0
+    llm_pos_ids_list = []
+    for modality_type, start_idx, end_idx in input_type_group:
+        if modality_type == 0:  # text
+            text_len = end_idx - start_idx
+            llm_pos_ids_list.append(
+                torch.arange(text_len, device=input_ids.device).view(1, -1).expand(3, -1) + current_pos
+            )
+            current_pos += text_len
+        else:  # image (1) or video (2)
+            grid_thw = next(grid_iters[modality_type])
+            vision_pos = get_vision_position_ids(
+                current_pos, grid_thw, spatial_merge_size, device=input_ids.device
+            )
+            llm_pos_ids_list.append(vision_pos)
+            # Qwen3.5 advances position by max(H, W) / spatial_merge_size (NOT T*H*W)
+            current_pos += max(grid_thw[1].item(), grid_thw[2].item()) // spatial_merge_size
+
+    llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+    if attention_mask is not None:
+        position_ids[..., attention_mask.bool()] = llm_positions.to(position_ids.device)
+    else:
+        position_ids = llm_positions.to(position_ids.device)
+
+    return position_ids

verl/utils/dataset.py

@@ -268,9 +268,32 @@ def __getitem__(self, index):
             input_ids = model_inputs.pop("input_ids")[0]
             attention_mask = model_inputs.pop("attention_mask")[0]
 
-        if self.processor is not None and "Qwen2VLImageProcessor" in self.processor.image_processor.__class__.__name__:
-            # qwen-vl mrope
-            if "Qwen3VLProcessor" in self.processor.__class__.__name__:
+        if self.processor is not None and self.processor.__class__.__name__ == "Qwen3_5_VLProcessor":
+            # Qwen3.5 uses mm_token_type_ids for mRoPE (different from Qwen2/3-VL's token-scanning approach)
+            from ..models.transformers.qwen3_5 import get_rope_index as get_rope_index_qwen35
+
+            mm_token_type_ids = model_inputs.get("mm_token_type_ids", None)
+            if mm_token_type_ids is not None:
+                mm_token_type_ids = mm_token_type_ids[0]  # remove batch dim
+            else:
+                # Construct from input_ids if processor didn't return it
+                mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int)
+                if hasattr(self.processor, "image_token_id"):
+                    mm_token_type_ids[input_ids == self.processor.image_token_id] = 1
+                if hasattr(self.processor, "video_token_id"):
+                    mm_token_type_ids[input_ids == self.processor.video_token_id] = 2
+
+            position_ids = get_rope_index_qwen35(
+                self.processor,
+                input_ids=input_ids,
+                mm_token_type_ids=mm_token_type_ids,
+                image_grid_thw=model_inputs.get("image_grid_thw", None),
+                video_grid_thw=model_inputs.get("video_grid_thw", None),
+                attention_mask=attention_mask,
+            )  # (3, seq_length)
+        elif self.processor is not None and "Qwen2VLImageProcessor" in self.processor.image_processor.__class__.__name__:
+            # Qwen2-VL / Qwen2.5-VL / Qwen3-VL use mRoPE with token-scanning
+            if self.processor.__class__.__name__ in ("Qwen3VLProcessor",):
                 from ..models.transformers.qwen3_vl import get_rope_index
             else:
                 from ..models.transformers.qwen2_vl import get_rope_index

verl/utils/flops_counter.py

@@ -71,6 +71,7 @@ def __init__(self, config: "LlamaConfig"):
             "qwen2_5_vl": self._estimate_llama_flops,
             "qwen3": self._estimate_llama_flops,
             "qwen3_vl": self._estimate_llama_flops,
+            "qwen3_5": self._estimate_llama_flops,
             "qwen3_moe": self._estimate_qwen2_moe_flops,
             "qwen3_vl_moe": self._estimate_qwen2_moe_flops,
         }

verl/utils/vllm_utils.py

@@ -17,7 +17,10 @@
 
 from msgspec import field
 from packaging import version as vs
-from vllm.lora.models import LoRAModel
+try:
+    from vllm.lora.models import LoRAModel
+except ImportError:
+    from vllm.lora.lora_model import LoRAModel
 from vllm.lora.request import LoRARequest
 from vllm.lora.utils import get_adapter_absolute_path
 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
@@ -72,38 +75,74 @@ def hijack__load_adapter(self, lora_request: TensorLoRARequest) -> LoRAModel:
             if hasattr(model, "hf_to_vllm_mapper") and model.hf_to_vllm_mapper is not None:
                 hf_to_vllm_mapper = model.hf_to_vllm_mapper
 
+            # vllm 0.17 changed API: target_embedding_padding -> model_vocab_size,
+            # removed embedding_modules/embedding_padding_modules
+            import inspect
+            _uses_new_api = "model_vocab_size" in inspect.signature(
+                self._lora_model_cls.from_local_checkpoint
+            ).parameters
+
             if isinstance(lora_request, TensorLoRARequest):
-                lora = self._lora_model_cls.from_lora_tensors(
-                    lora_model_id=lora_request.lora_int_id,
-                    tensors=lora_tensors,
-                    peft_helper=peft_helper,
-                    device="cpu",
-                    dtype=self.lora_config.lora_dtype,
-                    embeddings=None,
-                    target_embedding_padding=self.vocab_size + self.lora_config.lora_extra_vocab_size,
-                    embedding_modules=self.embedding_modules,
-                    embedding_padding_modules=self.embedding_padding_modules,
-                    weights_mapper=hf_to_vllm_mapper,
-                )
+                if _uses_new_api:
+                    lora = self._lora_model_cls.from_lora_tensors(
+                        lora_model_id=lora_request.lora_int_id,
+                        tensors=lora_tensors,
+                        peft_helper=peft_helper,
+                        device="cpu",
+                        dtype=self.lora_config.lora_dtype,
+                        model_vocab_size=self.vocab_size,
+                        weights_mapper=hf_to_vllm_mapper,
+                    )
+                else:
+                    extra_vocab = getattr(self.lora_config, 'lora_extra_vocab_size',
+                                          getattr(self.lora_config, 'max_extra_vocab_size', 0))
+                    lora = self._lora_model_cls.from_lora_tensors(
+                        lora_model_id=lora_request.lora_int_id,
+                        tensors=lora_tensors,
+                        peft_helper=peft_helper,
+                        device="cpu",
+                        dtype=self.lora_config.lora_dtype,
+                        embeddings=None,
+                        target_embedding_padding=self.vocab_size + extra_vocab,
+                        embedding_modules=self.embedding_modules,
+                        embedding_padding_modules=self.embedding_padding_modules,
+                        weights_mapper=hf_to_vllm_mapper,
+                    )
             else:
-                lora = self._lora_model_cls.from_local_checkpoint(
-                    lora_path,
-                    expected_lora_modules,
-                    peft_helper=peft_helper,
-                    lora_model_id=lora_request.lora_int_id,
-                    device="cpu",
-                    dtype=self.lora_config.lora_dtype,
-                    target_embedding_padding=self.vocab_size + self.lora_config.lora_extra_vocab_size,
-                    embedding_modules=self.embedding_modules,
-                    embedding_padding_modules=self.embedding_padding_modules,
-                    weights_mapper=hf_to_vllm_mapper,
-                )
-
-            if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
+                if _uses_new_api:
+                    lora = self._lora_model_cls.from_local_checkpoint(
+                        lora_path,
+                        expected_lora_modules,
+                        peft_helper=peft_helper,
+                        lora_model_id=lora_request.lora_int_id,
+                        device="cpu",
+                        dtype=self.lora_config.lora_dtype,
+                        model_vocab_size=self.vocab_size,
+                        weights_mapper=hf_to_vllm_mapper,
+                    )
+                else:
+                    extra_vocab = getattr(self.lora_config, 'lora_extra_vocab_size',
+                                          getattr(self.lora_config, 'max_extra_vocab_size', 0))
+                    lora = self._lora_model_cls.from_local_checkpoint(
+                        lora_path,
+                        expected_lora_modules,
+                        peft_helper=peft_helper,
+                        lora_model_id=lora_request.lora_int_id,
+                        device="cpu",
+                        dtype=self.lora_config.lora_dtype,
+                        target_embedding_padding=self.vocab_size + extra_vocab,
+                        embedding_modules=self.embedding_modules,
+                        embedding_padding_modules=self.embedding_padding_modules,
+                        weights_mapper=hf_to_vllm_mapper,
+                    )
+
+            lora_extra = getattr(lora, 'extra_vocab_size', 0)
+            config_extra = getattr(self.lora_config, 'lora_extra_vocab_size',
+                                   getattr(self.lora_config, 'max_extra_vocab_size', 0))
+            if lora_extra > config_extra:
                 raise ValueError(
-                    f"LoRA added vocab size {lora.extra_vocab_size} "
-                    f"is greater than lora_extra_vocab_size "
-                    f"{self.lora_config.lora_extra_vocab_size}."
+                    f"LoRA added vocab size {lora_extra} "
+                    f"is greater than max_extra_vocab_size {config_extra}."
                 )
             return lora
 

verl/workers/actor/config.py

@@ -26,6 +26,10 @@ class LoraConfig:
     alpha: int = 64
     target_modules: str = "all-linear"
     exclude_modules: Optional[str] = None
+    merge_for_rollout: bool = False
+    """Merge LoRA weights into the base model before syncing to vllm for rollouts.
+    Required for models with hybrid architectures (e.g. Qwen3.5 GDN layers)
+    where vllm's native LoRA mechanism cannot handle mixed layer dimensions."""
 
     def post_init(self):
         if not isinstance(self.target_modules, str):