具体报错如下：
Error executing job with overrides: ['algorithm.adv_estimator=grpo', 'data.train_files=data/train/one_shot_rlvr/dsr_sub.parquet', 'data.val_files=data/test/math500.parquet', 'data.train_batch_size=32', 'data.val_batch_size=128', 'data.max_prompt_length=1024', 'data.max_response_length=3072', 'reward_model.reward_manager=naive', 'actor_rollout_ref.model.path=/data/llm1/xinzhang68/project_files/One-Shot-RLVR-main/Qwen/Qwen2.5-Math-1.5B', 'actor_rollout_ref.actor.optim.lr=1e-6', 'actor_rollout_ref.model.use_remove_padding=True', 'actor_rollout_ref.rollout.enforce_eager=True', 'actor_rollout_ref.actor.ppo_mini_batch_size=32', 'actor_rollout_ref.actor.use_dynamic_bsz=True', 'actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8000', 'actor_rollout_ref.actor.use_kl_loss=True', 'actor_rollout_ref.actor.kl_loss_coef=0.001', 'actor_rollout_ref.actor.kl_loss_type=low_var_kl', 'actor_rollout_ref.model.enable_gradient_checkpointing=True', 'actor_rollout_ref.actor.fsdp_config.param_offload=False', '+actor_rollout_ref.actor.fsdp_config.grad_offload=False', 'actor_rollout_ref.actor.fsdp_config.optimizer_offload=False', 'actor_rollout_ref.rollout.tensor_model_parallel_size=1', 'actor_rollout_ref.rollout.name=vllm', 'actor_rollout_ref.rollout.temperature=0.6', '+actor_rollout_ref.rollout.val_temperature=0.6', 'actor_rollout_ref.rollout.gpu_memory_utilization=0.4', 'actor_rollout_ref.rollout.n=2', '+actor_rollout_ref.rollout.n_val=1', 'actor_rollout_ref.ref.fsdp_config.param_offload=True', 'algorithm.kl_ctrl.kl_coef=0.001', 'trainer.critic_warmup=0', 'trainer.logger=[console,wandb]', 'trainer.project_name=verl_few_shot', 'trainer.experiment_name=Qwen2.5-Math-1.5B-dsr_sub', 'trainer.checkpoints_dir=/data/llm1/xinzhang68/project_files/One-Shot-RLVR-main/checkpoints_dir', '+trainer.val_before_train=True', 'trainer.n_gpus_per_node=4', 'trainer.nnodes=1', 'trainer.save_freq=20', 'trainer.test_freq=20', 'trainer.default_hdfs_dir=null', 'trainer.total_epochs=2000']
Traceback (most recent call last):
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 24, in main
run_ppo(config)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 32, in run_ppo
ray.get(main_task.remote(config, compute_score))
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
return func(*args, **kwargs)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/worker.py", line 2981, in get
values, debugger_breakpoint = worker.get_objects(
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/worker.py", line 1012, in get_objects
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(RaySystemError): ray::main_task() (pid=127504, ip=172.31.101.237)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 130, in main_task
trainer.init_workers()
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/ppo/ray_trainer.py", line 802, in init_workers
wg_dict = self.ray_worker_group_cls(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 197, in init
self._init_with_resource_pool(resource_pool=resource_pool,
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 220, in _init_with_resource_pool
pgs = resource_pool.get_placement_groups(strategy=strategy)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 80, in get_placement_groups
pgs = [
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 81, in
placement_group(bundles=bundles, strategy=strategy, name=pg_name_prefix + str(idx), lifetime=lifetime)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/util/placement_group.py", line 208, in placement_group
placement_group_id = worker.core_worker.create_placement_group(
File "python/ray/includes/common.pxi", line 131, in ray._raylet.check_status
ray.exceptions.RaySystemError: System error: Failed to create placement group 'a8f9e41eb8189d511b2320d9263601000000' because name 'global_poolverl_group_4:0' already exists.
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
安装训练环境后，执行脚本 一直出现 提示
具体报错如下：
Error executing job with overrides: ['algorithm.adv_estimator=grpo', 'data.train_files=data/train/one_shot_rlvr/dsr_sub.parquet', 'data.val_files=data/test/math500.parquet', 'data.train_batch_size=32', 'data.val_batch_size=128', 'data.max_prompt_length=1024', 'data.max_response_length=3072', 'reward_model.reward_manager=naive', 'actor_rollout_ref.model.path=/data/llm1/xinzhang68/project_files/One-Shot-RLVR-main/Qwen/Qwen2.5-Math-1.5B', 'actor_rollout_ref.actor.optim.lr=1e-6', 'actor_rollout_ref.model.use_remove_padding=True', 'actor_rollout_ref.rollout.enforce_eager=True', 'actor_rollout_ref.actor.ppo_mini_batch_size=32', 'actor_rollout_ref.actor.use_dynamic_bsz=True', 'actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8000', 'actor_rollout_ref.actor.use_kl_loss=True', 'actor_rollout_ref.actor.kl_loss_coef=0.001', 'actor_rollout_ref.actor.kl_loss_type=low_var_kl', 'actor_rollout_ref.model.enable_gradient_checkpointing=True', 'actor_rollout_ref.actor.fsdp_config.param_offload=False', '+actor_rollout_ref.actor.fsdp_config.grad_offload=False', 'actor_rollout_ref.actor.fsdp_config.optimizer_offload=False', 'actor_rollout_ref.rollout.tensor_model_parallel_size=1', 'actor_rollout_ref.rollout.name=vllm', 'actor_rollout_ref.rollout.temperature=0.6', '+actor_rollout_ref.rollout.val_temperature=0.6', 'actor_rollout_ref.rollout.gpu_memory_utilization=0.4', 'actor_rollout_ref.rollout.n=2', '+actor_rollout_ref.rollout.n_val=1', 'actor_rollout_ref.ref.fsdp_config.param_offload=True', 'algorithm.kl_ctrl.kl_coef=0.001', 'trainer.critic_warmup=0', 'trainer.logger=[console,wandb]', 'trainer.project_name=verl_few_shot', 'trainer.experiment_name=Qwen2.5-Math-1.5B-dsr_sub', 'trainer.checkpoints_dir=/data/llm1/xinzhang68/project_files/One-Shot-RLVR-main/checkpoints_dir', '+trainer.val_before_train=True', 'trainer.n_gpus_per_node=4', 'trainer.nnodes=1', 'trainer.save_freq=20', 'trainer.test_freq=20', 'trainer.default_hdfs_dir=null', 'trainer.total_epochs=2000']
Traceback (most recent call last):
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 24, in main
run_ppo(config)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 32, in run_ppo
ray.get(main_task.remote(config, compute_score))
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
return func(*args, **kwargs)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/worker.py", line 2981, in get
values, debugger_breakpoint = worker.get_objects(
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/_private/worker.py", line 1012, in get_objects
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(RaySystemError): ray::main_task() (pid=127504, ip=172.31.101.237)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/main_ppo.py", line 130, in main_task
trainer.init_workers()
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/trainer/ppo/ray_trainer.py", line 802, in init_workers
wg_dict = self.ray_worker_group_cls(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 197, in init
self._init_with_resource_pool(resource_pool=resource_pool,
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 220, in _init_with_resource_pool
pgs = resource_pool.get_placement_groups(strategy=strategy)
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 80, in get_placement_groups
pgs = [
File "/data02/llm1/xinzhang68/project_files/One-Shot-RLVR-main/verl/single_controller/ray/base.py", line 81, in
placement_group(bundles=bundles, strategy=strategy, name=pg_name_prefix + str(idx), lifetime=lifetime)
File "/data/llm1/xinzhang68/miniforge3/envs/rlvr_train/lib/python3.10/site-packages/ray/util/placement_group.py", line 208, in placement_group
placement_group_id = worker.core_worker.create_placement_group(
File "python/ray/includes/common.pxi", line 131, in ray._raylet.check_status
ray.exceptions.RaySystemError: System error: Failed to create placement group 'a8f9e41eb8189d511b2320d9263601000000' because name 'global_poolverl_group_4:0' already exists.
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
安装训练环境后，执行脚本 一直出现 提示