Hello, when I use GRPO and Qwen2.5-VL-3B for multi-node training, I noticed that the time/step does not decrease linearly as the number of nodes increases. What could be the reason for this?
I used 2 and 8 nodes for training and the time is shown as:
This is my config:
`data:
train_files: train
val_files: test
prompt_key: problem
answer_key: answer
image_key: images
video_key: videos
image_dir: null
video_fps: 2.0
max_prompt_length: 2048
max_response_length: 2048
rollout_batch_size: 512 # equivalent to verl's data.train_batch_size
mini_rollout_batch_size: null # equivalent to verl's data.gen_batch_size
val_batch_size: 1024
format_prompt: examples/format_prompt/math.jinja
override_chat_template: null
shuffle: true
seed: 1
max_pixels: 1003520 # 12802828
min_pixels: 200704 # 2562828
filter_overlong_prompts: true
algorithm:
adv_estimator: grpo
disable_kl: false
use_kl_loss: true
kl_penalty: low_var_kl
kl_coef: 1.0e-2
online_filtering: false # dapo filter groups
filter_key: overall
filter_low: 0.01
filter_high: 0.99
use_ori_entropy_loss: false
ori_entropy_loss_coef: 1.0e-5
worker:
actor:
global_batch_size: 128 # equivalent to verl's actor.ppo_mini_batch_size
micro_batch_size_per_device_for_update: 4 # equivalent to verl's actor.ppo_micro_batch_size_per_gpu
micro_batch_size_per_device_for_experience: 16 # equivalent to verl's rollout.log_prob_micro_batch_size_per_gpu
max_grad_norm: 1.0
padding_free: true
ulysses_size: 1
model:
model_path: Qwen/Qwen2.5-3B-Instruct
enable_gradient_checkpointing: true
trust_remote_code: true
freeze_vision_tower: false
optim:
lr: 1.0e-6
weight_decay: 1.0e-2
strategy: adamw_bf16
lr_warmup_ratio: 0.0
fsdp:
torch_dtype: bf16
enable_full_shard: true
enable_cpu_offload: false
enable_rank0_init: true
offload:
offload_params: true # true: more CPU memory; false: more GPU memory
offload_optimizer: true # true: more CPU memory; false: more GPU memory
rollout:
n: 5
temperature: 1.0
top_p: 0.99
limit_images: 10
gpu_memory_utilization: 0.8
enforce_eager: false
enable_chunked_prefill: false
tensor_parallel_size: 1
disable_tqdm: false
val_override_config:
temperature: 1.0
n: 8 # avg acc @ 8
ref:
fsdp:
torch_dtype: bf16
enable_full_shard: true
enable_cpu_offload: true # true: more CPU memory; false: more GPU memory
enable_rank0_init: true
offload:
offload_params: false
reward:
reward_type: batch
reward_function: examples/reward_function/math.py:compute_score
trainer:
total_epochs: 15
max_steps: null
project_name: easy_r1
experiment_name: qwen2_5_3b_grpo
logger: ["console", "wandb"]
nnodes: 1 # I have set it to 1 and 8
n_gpus_per_node: 4
max_try_make_batch: 20 # -1 means no limit
val_freq: 5 # -1 to disable
val_before_train: true
val_only: false
val_generations_to_log: 3
save_freq: 5 # -1 to disable
save_limit: 3 # -1 to disable
save_model_only: false
save_checkpoint_path: null
load_checkpoint_path: null
save_best_checkpoint: true
find_last_checkpoint: true
`
Hello, when I use GRPO and Qwen2.5-VL-3B for multi-node training, I noticed that the time/step does not decrease linearly as the number of nodes increases. What could be the reason for this?
I used 2 and 8 nodes for training and the time is shown as:
This is my config:
`data:
train_files: train
val_files: test
prompt_key: problem
answer_key: answer
image_key: images
video_key: videos
image_dir: null
video_fps: 2.0
max_prompt_length: 2048
max_response_length: 2048
rollout_batch_size: 512 # equivalent to verl's data.train_batch_size
mini_rollout_batch_size: null # equivalent to verl's data.gen_batch_size
val_batch_size: 1024
format_prompt: examples/format_prompt/math.jinja
override_chat_template: null
shuffle: true
seed: 1
max_pixels: 1003520 # 12802828
min_pixels: 200704 # 2562828
filter_overlong_prompts: true
algorithm:
adv_estimator: grpo
disable_kl: false
use_kl_loss: true
kl_penalty: low_var_kl
kl_coef: 1.0e-2
online_filtering: false # dapo filter groups
filter_key: overall
filter_low: 0.01
filter_high: 0.99
use_ori_entropy_loss: false
ori_entropy_loss_coef: 1.0e-5
worker:
actor:
global_batch_size: 128 # equivalent to verl's actor.ppo_mini_batch_size
micro_batch_size_per_device_for_update: 4 # equivalent to verl's actor.ppo_micro_batch_size_per_gpu
micro_batch_size_per_device_for_experience: 16 # equivalent to verl's rollout.log_prob_micro_batch_size_per_gpu
max_grad_norm: 1.0
padding_free: true
ulysses_size: 1
model:
model_path: Qwen/Qwen2.5-3B-Instruct
enable_gradient_checkpointing: true
trust_remote_code: true
freeze_vision_tower: false
optim:
lr: 1.0e-6
weight_decay: 1.0e-2
strategy: adamw_bf16
lr_warmup_ratio: 0.0
fsdp:
torch_dtype: bf16
enable_full_shard: true
enable_cpu_offload: false
enable_rank0_init: true
offload:
offload_params: true # true: more CPU memory; false: more GPU memory
offload_optimizer: true # true: more CPU memory; false: more GPU memory
rollout:
n: 5
temperature: 1.0
top_p: 0.99
limit_images: 10
gpu_memory_utilization: 0.8
enforce_eager: false
enable_chunked_prefill: false
tensor_parallel_size: 1
disable_tqdm: false
val_override_config:
temperature: 1.0
n: 8 # avg acc @ 8
ref:
fsdp:
torch_dtype: bf16
enable_full_shard: true
enable_cpu_offload: true # true: more CPU memory; false: more GPU memory
enable_rank0_init: true
offload:
offload_params: false
reward:
reward_type: batch
reward_function: examples/reward_function/math.py:compute_score
trainer:
total_epochs: 15
max_steps: null
project_name: easy_r1
experiment_name: qwen2_5_3b_grpo
logger: ["console", "wandb"]
nnodes: 1 # I have set it to 1 and 8
n_gpus_per_node: 4
max_try_make_batch: 20 # -1 means no limit
val_freq: 5 # -1 to disable
val_before_train: true
val_only: false
val_generations_to_log: 3
save_freq: 5 # -1 to disable
save_limit: 3 # -1 to disable
save_model_only: false
save_checkpoint_path: null
load_checkpoint_path: null
save_best_checkpoint: true
find_last_checkpoint: true
`