Feature/find ppo error

evals/redis_stat.py

@@ -129,4 +129,4 @@ def analyze_episodes_with_positions(tag):
     }
 
 # Run the analysis
-results = analyze_episodes_with_positions("Qwen2.5-7B-Instruct_vs_sft_qwen25_7b_bigtom_step_1500-bigtom_0402")
+results = analyze_episodes_with_positions("grpo_direct_step_400_vs_sft_qwen25_7b_sft_round_1_bc_data_top_2_step_1500-0420")

scripts/accelerate_config_ppo.yaml

@@ -2,11 +2,6 @@ compute_environment: LOCAL_MACHINE
 debug: true
 distributed_type: MULTI_GPU
 downcast_bf16: 'no'
-dynamo_config:
-  dynamo_backend: EAGER
-  dynamo_mode: default
-  dynamo_use_dynamic: false
-  dynamo_use_fullgraph: false
 enable_cpu_affinity: true
 gpu_ids: all
 machine_rank: 0

scripts/accelerate_config_rm.yaml

@@ -8,7 +8,7 @@ machine_rank: 0
 main_training_function: main
 mixed_precision: bf16
 num_machines: 1
-num_processes: 5
+num_processes: 8
 rdzv_backend: static
 same_network: true
 tpu_env: []

scripts/accelerate_config_sft.yaml

@@ -8,7 +8,7 @@ machine_rank: 0
 main_training_function: main
 mixed_precision: bf16
 num_machines: 1
-num_processes: 2 
+num_processes: 8
 rdzv_backend: static
 same_network: true
 tpu_env: []

scripts/inference_rm.sh

@@ -8,8 +8,8 @@ CUDA_VISIBLE_DEVICES=9 python inference_rm.py \
   --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json"
 
 
-CUDA_VISIBLE_DEVICES=8 python inference_rm.py \
+CUDA_VISIBLE_DEVICES=5 python inference_rm.py \
   --model_path "/mnt/data_from_server1/models/Qwen2.5-7B-Instruct" \
-  --adapter_path "/data/haofeiy2/sotopia-rl/rm_overfit_test/checkpoint-100" \
+  --adapter_path "/data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800" \
   --template_path "/data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja" \
   --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json"

scripts/train_grpo.py

@@ -31,6 +31,8 @@
                         help="Maximum length of generated responses")
     parser.add_argument("--num_generations", type=int, default=4,
                         help="Number of generations for GRPO")
+    parser.add_argument("--beta", type=float, default=0.04,
+                        help="KL coefficient for GRPO")
 
     # Adapter parameters
     parser.add_argument("--policy_adapter_path", type=str, default=None,

scripts/train_grpo.sh

@@ -1,4 +1,4 @@
-CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 accelerate launch \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \
   --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_grpo.yaml \
   --main_process_port 29511 \
   /data/disk0/sotopia-rl/scripts/train_grpo.py \
@@ -14,4 +14,9 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 accelerate launch \
   --num_grpo_epochs 2 \
   --use_lora_train_grpo \
   --num_generations 16 \
+<<<<<<< HEAD
+  --beta 0.04 \
+  --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default_beta_004
+=======
   --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default
+>>>>>>> 6a5e62f4de6e9b376b6ed3f56e6745b8bb0f86bf

scripts/train_ppo.py

@@ -20,7 +20,7 @@
                         help="Number of PPO epochs per update")
     parser.add_argument("--learning_rate", type=float, default=5e-6,
                         help="Learning rate for optimizer")
-    parser.add_argument("--gamma", type=float, default=0.99,
+    parser.add_argument("--gamma", type=float, default=1.0,
                         help="Discount factor")
     parser.add_argument("--lam", type=float, default=0.95,
                         help="GAE lambda for advantage estimation")
@@ -32,7 +32,7 @@
                         help="Number of steps to accumulate gradients before performing an update")
     parser.add_argument("--val_ratio", type=float, default=0.05,
                         help="Ratio of validation data")
-    parser.add_argument("--response_length", type=int, default=128,
+    parser.add_argument("--response_length", type=int, default=256,
                         help="Maximum length of generated responses")
     parser.add_argument("--local_rollout_forward_batch_size", type=int, default=16,
                         help="Batch size for local rollout forward pass")
@@ -69,6 +69,5 @@
                         help="Use LoRA for training PPO")
 
     args = parser.parse_args()
-    accelerator = Accelerator()
-    trainer = SotopiaPPOTrainer(args, accelerator)
+    trainer = SotopiaPPOTrainer(args)
     trainer.train()

scripts/train_ppo.sh

@@ -1,21 +1,213 @@
-CUDA_VISIBLE_DEVICES=1,7,8 accelerate launch \
-  --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config.yaml \
-  --main_process_port 29511 \
-  /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \
-  --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \
-  --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
-  --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
-  --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \
-  --value_adapter_path /data/haofeiy2/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \
+# parameter I used for final PPO checkpoint
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29439 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm/checkpoint-90 \
+  --learning_rate 3e-6 \
+  --per_device_train_batch_size 3 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 4 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 30 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 0.99 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model_gamma_099_direct_rm
+
+# param for pretrain value adapter
+CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29439 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \
+  --learning_rate 5e-5 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 4 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 30 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm
+
+
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29439 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_checkpoint-800 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_pretrained_value_model \
   --learning_rate 1e-5 \
   --per_device_train_batch_size 1 \
-  --per_device_eval_batch_size 1 \
-  --gradient_accumulation_steps 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 2 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_with_pretrained_value_model
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29449 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 4 \
   --num_mini_batches 1 \
-  --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
-  --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
   --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step30_default_kl
+
+
+CUDA_VISIBLE_DEVICES=2,3 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29469 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 4 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
   --num_train_epochs 5 \
-  --gamma 0.99 \
-  --lam 0.95 \
-  --output_dir /data/haofeiy2/sotopia-rl/ppo_origin_qwen25_7b_reward_direct_default_no_goal_gpt-4o_without_goal_leak_with_sft_self_play_data_use_sotopia_pi_full_data_0408
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step50_default_kl
+
+
+CUDA_VISIBLE_DEVICES=0,1 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29499 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 4 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step70_default_kl
+
+
+CUDA_VISIBLE_DEVICES=5 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29549 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 32 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/
+
+CUDA_VISIBLE_DEVICES=6 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29559 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 32 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt500
+
+CUDA_VISIBLE_DEVICES=7 accelerate launch \
+  --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \
+  --main_process_port 29569 \
+ /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \
+  --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \
+  --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \
+  --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \
+  --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \
+  --learning_rate 1e-6 \
+  --per_device_train_batch_size 1 \
+  --per_device_eval_batch_size 4 \
+  --gradient_accumulation_steps 32 \
+  --num_mini_batches 1 \
+  --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \
+  --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --num_train_epochs 5 \
+  --max_length 4096 \
+  --num_ppo_epochs 2 \
+  --gamma 1.00 \
+  --use_lora_train_ppo \
+  --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt700

scripts/train_rm.sh

@@ -1,18 +1,18 @@
-CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \
-  --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \
+  --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_rm.yaml \
   --main_process_port 29500 \
-  /data/haofeiy2/sotopia-rl/scripts/train_rm.py \
-  --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \
+  /data/disk0/sotopia-rl/scripts/train_rm.py \
+  --model_name /data/disk0/models/Qwen2.5-7B-Instruct \
   --learning_rate 1e-5 \
   --max_length 4096 \
-  --train_batch_size 1 \
+  --train_batch_size 4 \
   --val_batch_size 1 \
-  --accumulation_steps 8 \
+  --accumulation_steps 2 \
   --num_epochs 30 \
   --evaluation_steps 50 \
-  --reward_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length.json \
-  --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \
-  --checkpoint_dir /data/haofeiy2/sotopia-rl/rm_token_length
+  --reward_data_path /data/disk0/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length_binary.json \
+  --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \
+  --checkpoint_dir /data/disk0/sotopia-rl/rm_token_length_binary
 
 CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \
   --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \