[FEAT] Support GLM4.1V series

examples/glm4.1v_9b_thinking_geo3k_grpo.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x
+
+MODEL_PATH=zai-org/GLM-4.1V-9B-Thinking  # replace it with your local file path
+
+python3 -m verl.trainer.main \
+    config=examples/config.yaml \
+    data.train_files=hiyouga/geometry3k@train \
+    data.val_files=hiyouga/geometry3k@test \
+    worker.actor.model.model_path=${MODEL_PATH} \
+    trainer.experiment_name=glm4.1v_thinking_geo_grpo \
+    worker.actor.padding_free=False \
+    trainer.n_gpus_per_node=8

examples/glm4.1v_base_geo3k_grpo.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x
+
+MODEL_PATH=zai-org/GLM-4.1V-9B-Base  # replace it with your local file path
+
+python3 -m verl.trainer.main \
+    config=examples/config.yaml \
+    data.train_files=hiyouga/geometry3k@train \
+    data.val_files=hiyouga/geometry3k@test \
+    worker.actor.model.model_path=${MODEL_PATH} \
+    trainer.experiment_name=glm4.1v_base_geo_grpo \
+    worker.actor.padding_free=False \
+    trainer.n_gpus_per_node=8

verl/models/monkey_patch.py

@@ -35,6 +35,8 @@
 
 QWEN2_VL_MODELS = ("qwen2_vl", "qwen2_5_vl")
 QWEN3_VL_MODELS = ("qwen3_vl", "qwen3_vl_moe")
+# TODO support monkey patch for glm4.1v
+GLM_VL_MODELS = ("glm4.1v_base", "glm4.1v_thinking")
 
 
 def apply_ulysses_patch(model_type: str) -> None:
@@ -44,7 +46,7 @@ def apply_ulysses_patch(model_type: str) -> None:
     if model_type in SUPPORTED_MODEL_TYPE:
         ALL_ATTENTION_FUNCTIONS["flash_attention_2"] = flash_attention_forward
     else:
-        raise NotImplementedError(f"Model architecture {model_type} is not supported yet.")
+        raise NotImplementedError(f"Model architecture {model_type} is not supported ulysses_patch (patch_free) yet.")
 
     if model_type in QWEN2_VL_MODELS:
         from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (

verl/models/transformers/glm4v.py

@@ -0,0 +1,165 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import itertools
+import logging
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+from transformers.modeling_flash_attention_utils import _flash_attention_forward, fa_peft_integration_check
+from transformers.models.glm4v.modeling_glm4v import (
+    Glm4vCausalLMOutputWithPast,
+    Glm4vForConditionalGeneration,
+    Glm4vTextAttention,
+)
+from transformers.utils import is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10
+
+# from verl.utils.device import is_npu_available
+from verl.utils.ulysses import (
+    gather_heads_scatter_seq,
+    gather_seq_scatter_heads,
+    get_ulysses_sequence_parallel_group,
+    get_ulysses_sequence_parallel_world_size,
+    validate_ulysses_config,
+)
+
+logger = logging.getLogger(__file__)
+logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN"))
+
+
+
+def get_rope_index(
+    processor,
+    input_ids: torch.Tensor,
+    image_grid_thw: Optional[torch.LongTensor] = None,
+    video_grid_thw: Optional[torch.LongTensor] = None,
+    attention_mask: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Gets the position ids for GLM4V in padding-free format.
+    The batch dim has been removed and the input_ids should be a 1D tensor representing a single example.
+    """
+    spatial_merge_size = processor.image_processor.merge_size
+    image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image|>")
+    video_start_token_id = processor.tokenizer.convert_tokens_to_ids("<|begin_of_video|>")
+    video_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|end_of_video|>")
+
+    if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None):
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+
+        position_ids = torch.ones(3, input_ids.size(0), dtype=input_ids.dtype, device=input_ids.device)  # (3, seqlen)
+        image_index, video_index = 0, 0
+        video_group_index = 0
+
+        input_ids_filtered = input_ids[attention_mask == 1]
+        input_tokens = input_ids_filtered.tolist()
+
+        input_token_type = []
+        video_check_flg = False
+        for token in input_tokens:
+            if token == video_start_token_id:
+                video_check_flg = True
+            elif token == video_end_token_id:
+                video_check_flg = False
+
+            if token == image_token_id and not video_check_flg:
+                input_token_type.append("image")
+            elif token == image_token_id and video_check_flg:
+                input_token_type.append("video")
+            else:
+                input_token_type.append("text")
+
+        input_type_group = []
+        for key, group in itertools.groupby(enumerate(input_token_type), lambda x: x[1]):
+            group = list(group)
+            start_index = group[0][0]
+            end_index = group[-1][0] + 1
+            input_type_group.append((key, start_index, end_index))
+
+        llm_pos_ids_list = []
+        video_frame_num = 1
+
+        for modality_type, start_idx, end_idx in input_type_group:
+            st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
+
+            if modality_type == "image":
+                t, h, w = (
+                    image_grid_thw[image_index][0],
+                    image_grid_thw[image_index][1],
+                    image_grid_thw[image_index][2],
+                )
+                llm_grid_t, llm_grid_h, llm_grid_w = (
+                    t.item(),
+                    h.item() // spatial_merge_size,
+                    w.item() // spatial_merge_size,
+                )
+
+                t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten()
+                h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten()
+                w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten()
+                llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx)
+
+                image_index += 1
+                video_frame_num = 1
+
+            elif modality_type == "video":
+                t, h, w = (
+                    video_frame_num,
+                    video_grid_thw[video_index][1],
+                    video_grid_thw[video_index][2],
+                )
+
+                llm_grid_t, llm_grid_h, llm_grid_w = (
+                    t,
+                    h.item() // spatial_merge_size,
+                    w.item() // spatial_merge_size,
+                )
+
+                for t_idx in range(llm_grid_t):
+                    t_index = torch.tensor(t_idx).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten()
+                    h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(1, -1, llm_grid_w).flatten()
+                    w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(1, llm_grid_h, -1).flatten()
+                    llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + st_idx)
+
+                video_group_index += 1
+
+                if video_group_index >= video_grid_thw[video_index][0]:
+                    video_index += 1
+                    video_group_index = 0
+
+                video_frame_num += 1
+
+            else:
+                text_len = end_idx - start_idx
+                llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
+                video_frame_num = 1
+
+        llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
+        position_ids[..., attention_mask == 1] = llm_positions.to(position_ids.device)
+    else:
+        if attention_mask is not None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            position_ids = position_ids.unsqueeze(0).expand(3, -1).to(input_ids.device)
+        else:
+            position_ids = torch.arange(input_ids.shape[0], device=input_ids.device).view(1, -1).expand(3, -1)
+
+    return position_ids
+
+

verl/utils/dataset.py

@@ -280,6 +280,24 @@ def __getitem__(self, index):
             )  # (3, seq_length)
             text_position_ids = torch.arange(len(input_ids)).unsqueeze(0)  # (1, seq_length)
             position_ids = torch.cat((text_position_ids, vision_position_ids), dim=0)  # (4, seq_length)
+
+        elif self.processor is not None and "Glm4vImageProcessor" in self.processor.image_processor.__class__.__name__:
+            from verl.models.transformers.glm4v import get_rope_index
+
+            vision_position_ids = get_rope_index(
+                self.processor,
+                input_ids=input_ids,
+                image_grid_thw=model_inputs.get("image_grid_thw", None),
+                video_grid_thw=model_inputs.get("video_grid_thw", None),
+                attention_mask=attention_mask,
+            )  # (3, seq_length)
+
+            text_position_ids = torch.arange(len(input_ids))
+            text_position_ids=text_position_ids.unsqueeze(0)  # (1, seq_length)
+            # GLM4V only needs 3D vision position_ids for rotary embedding
+            # Use only vision_position_ids instead of concatenating with text_position_ids
+            position_ids = vision_position_ids  # (3, seq_length)
+
         else:
             position_ids = torch.clip(attention_mask.cumsum(dim=0) - 1, min=0, max=None)  # (seq_length,)
 

verl/workers/fsdp_workers.py

@@ -478,7 +478,14 @@ def _process_multi_modal_inputs(self, data: DataProto):
                         )
                     else:
                         multi_modal_inputs = {}
-
+
+                    if "pixel_values" in multi_modal_inputs and multi_modal_inputs["pixel_values"].ndim == 3:
+                        # Some image processor return with batch dim (such as glm4.1), we need to squeeze the pix_value.
+                        # i.e. (1,patch,pix_per_patch) -> (patch,pix_per_patch)
+                        multi_modal_inputs["pixel_values"] = multi_modal_inputs["pixel_values"].squeeze(0)
+
+
+
                     multi_modal_inputs_cache[index] = multi_modal_inputs
 
                 batch_multi_modal_inputs.append(multi_modal_inputs_cache[index])