From 03651b000b386d76d9a8ac28004455abf1a1bd7f Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 19 Apr 2025 02:48:27 -0500 Subject: [PATCH 01/26] bug(rm): inference not correct --- scripts/inference_rm.py | 100 +++++++++++++++++++++-------- sotopia_rl/rm_trainer.py | 133 ++++++++++++++++++++++++++++----------- 2 files changed, 171 insertions(+), 62 deletions(-) diff --git a/scripts/inference_rm.py b/scripts/inference_rm.py index 188b35a..4bcbb3d 100644 --- a/scripts/inference_rm.py +++ b/scripts/inference_rm.py @@ -12,36 +12,69 @@ def parse_args(): parser = argparse.ArgumentParser( description="Test a reward model with a template and example data" ) - parser.add_argument("--model_path", type=str, required=True, help="Path to base model or HF model name") - parser.add_argument("--adapter_path", type=str, required=True, help="Path to saved checkpoint directory") - parser.add_argument("--template_path", type=str, required=True, help="Path to Jinja template file") - parser.add_argument("--example_path", type=str, required=True, help="Path to example data JSON") + parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to base model or HF model name", + ) + parser.add_argument( + "--adapter_path", + type=str, + required=True, + help="Path to saved checkpoint directory", + ) + parser.add_argument( + "--template_path", type=str, required=True, help="Path to Jinja template file" + ) + parser.add_argument( + "--example_path", type=str, required=True, help="Path to example data JSON" + ) return parser.parse_args() + def load_model_and_tokenizer(args): print(f"Loading base model: {args.model_path}") tokenizer = AutoTokenizer.from_pretrained(args.model_path) print("Using full precision model") - base_model = AutoModelForSequenceClassification.from_pretrained( - args.model_path, - torch_dtype=torch.float32, #important + model = AutoModelForSequenceClassification.from_pretrained( + args.adapter_path, + # torch_dtype=torch.float32, #important + torch_dtype="auto", device_map="auto", num_labels=1, # For regression task - pad_token_id=tokenizer.pad_token_id # very important to add this + pad_token_id=tokenizer.pad_token_id, # very important to add this ) - adapter_path = os.path.join(args.adapter_path, 'adapter_model') - if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): - print(f"Loading adapter from: {args.adapter_path}") - model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) - else: - print(f"No adapter found at {adapter_path}, using base model") - model = base_model + # adapter_path = os.path.join(args.adapter_path, 'adapter_model') + # # model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) + # if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + # print(f"Loading adapter from: {args.adapter_path}") + # model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) + # else: + # print(f"No adapter found at {adapter_path}, using base model") + # model = base_model + # print("Active adapters:", model.active_adapters) + # score_weights = model.base_model.model.score.weight.data.view(-1) + # print("Mean of score.weight:", score_weights.mean().item()) + # print("Std of score.weight:", score_weights.std().item()) + # print("Model weight:", model.score.weight) + def print_named_parameters(model, keyword="score"): + for name, param in model.named_parameters(): + if keyword in name: + print( + f"{name}: mean={param.data.mean():.4f}, std={param.data.std():.4f}" + ) + else: + print("did not load score_weights") + + print_named_parameters(model, keyword="score") model.eval() return model, tokenizer + def load_template(template_path): template_dir = os.path.dirname(template_path) template_file = os.path.basename(template_path) @@ -50,29 +83,42 @@ def load_template(template_path): template_dir = "." env = Environment(loader=FileSystemLoader(template_dir)) - env.filters['tojson'] = lambda obj: json.dumps(obj) + env.filters["tojson"] = lambda obj: json.dumps(obj) return env.get_template(template_file) -def evaluate_prompt(model, tokenizer, prompt): - inputs = tokenizer(prompt, return_tensors="pt", truncation=True) + +def evaluate_prompt(model, tokenizer, prompt, index=None): + print(f"\n[DEBUG] Prompt [{index}]:") + print(prompt) + print("[DEBUG] Decoded Input IDs:") + encoded = tokenizer(prompt, return_tensors="pt", truncation=True) + print(tokenizer.decode(encoded["input_ids"][0], skip_special_tokens=False)) + + # Check input length + print(f"[DEBUG] Input length: {encoded['input_ids'].shape[-1]} tokens") + # inputs = tokenizer(prompt, return_tensors="pt", truncation=True) device = next(model.parameters()).device - inputs = {k: v.to(device) for k, v in inputs.items()} + inputs = {k: v.to(device) for k, v in encoded.items()} with torch.no_grad(): outputs = model(**inputs) # Get reward score directly from the logits - reward = outputs.logits.squeeze().cpu().item() + # reward = outputs.logits.squeeze().cpu().item() + logits = outputs.logits.squeeze() + print(f"[DEBUG] Raw logits: {logits}") + reward = logits.cpu().item() return reward + def main(): args = parse_args() model, tokenizer = load_model_and_tokenizer(args) - with open(args.example_path, 'r') as f: + with open(args.example_path, "r") as f: example_data = json.load(f) template = load_template(args.template_path) @@ -81,14 +127,15 @@ def main(): rendered_prompt = template.render( messages=[ - {"role": "user", "content": example['input']}, - {"role": "assistant", "content": example['output']}, + {"role": "user", "content": example["input"]}, + {"role": "assistant", "content": example["output"]}, ], - add_generation_prompt=False + add_generation_prompt=False, ) - reward = evaluate_prompt(model, tokenizer, rendered_prompt) - gth_reward = example.get('value') + # reward = evaluate_prompt(model, tokenizer, rendered_prompt) + reward = evaluate_prompt(model, tokenizer, rendered_prompt, index=i + 1) + gth_reward = example.get("value") print(f"REWARD SCORE: {reward:.6f}") if gth_reward is not None: @@ -96,5 +143,6 @@ def main(): else: print("GTH REWARD: Not available") + if __name__ == "__main__": main() diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index 85da979..c221884 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -3,7 +3,7 @@ import torch import torch.distributed as dist from jinja2 import Environment, FileSystemLoader -from peft import LoraConfig, get_peft_model +from peft import LoraConfig, get_peft_model, PeftModelForSequenceClassification from torch.nn import MSELoss from torch.utils.data import random_split from transformers import ( @@ -13,13 +13,15 @@ TrainingArguments, ) from accelerate import Accelerator +from typing import Optional import wandb from sotopia_rl.data import RMDataset import torch._dynamo + torch._dynamo.config.suppress_errors = True -os.environ['NCCL_P2P_DISABLE'] = '1' +os.environ["NCCL_P2P_DISABLE"] = "1" os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" @@ -33,30 +35,44 @@ def __init__(self, args, accelerator, **kwargs): train_dataset, eval_dataset = self.setup_dataset(tokenizer) # Initialize wandb only on the main process - if self.accelerator.is_main_process: - wandb.init( - project=args.wandb_project, - name=args.wandb_run_name, - config={k: v for k, v in vars(args).items() if isinstance(v, (int, float, str))} - ) + # if self.accelerator.is_main_process: + wandb.init( + project=args.wandb_project, + name=args.wandb_run_name, + config={ + k: v for k, v in vars(args).items() if isinstance(v, (int, float, str)) + }, + ) peft_config = LoraConfig( r=args.lora_r, lora_alpha=args.lora_alpha, lora_dropout=args.lora_dropout, - target_modules=args.target_modules.split(",") + target_modules=args.target_modules.split(","), ) base_model = AutoModelForSequenceClassification.from_pretrained( args.model_name, num_labels=1, - torch_dtype='auto', + torch_dtype="auto", ) - model = get_peft_model(base_model, peft_config) - model.config.pad_token_id = tokenizer.pad_token_id # important to set the config pad_token_id - - model = self.accelerator.prepare_model(model) + # self.model = get_peft_model(base_model, peft_config) + self.model = PeftModelForSequenceClassification(base_model, peft_config) + param_check = self.model.base_model.model.score.weight + print( + f"whether score.weight is trainable: {param_check.requires_grad}, shape: {param_check.shape}" + ) + print(f"mean={param_check.data.mean():.4f}, std={param_check.data.std():.4f}") + count = 0 + for name, param in self.model.named_parameters(): + if param.requires_grad: + count += 1 + print(f"{name} shape={param.shape}") + print(f"Total trainable parameters: {count}") + self.model.config.pad_token_id = ( + tokenizer.pad_token_id + ) # important to set the config pad_token_id # Set up the TrainingArguments with DeepSpeed support training_args = TrainingArguments( @@ -66,6 +82,7 @@ def __init__(self, args, accelerator, **kwargs): num_train_epochs=args.num_epochs, logging_steps=1, save_steps=args.evaluation_steps, + save_strategy="steps", eval_steps=args.evaluation_steps, logging_dir="./logs", gradient_accumulation_steps=args.accumulation_steps, @@ -78,26 +95,49 @@ def __init__(self, args, accelerator, **kwargs): ddp_find_unused_parameters=False, ) - collate_fn = train_dataset.dataset.collate_fn if hasattr(train_dataset, 'dataset') else None + print("Trainable parameters:") + + collate_fn = ( + train_dataset.dataset.collate_fn + if hasattr(train_dataset, "dataset") + else None + ) super().__init__( - model=model, + model=self.model, args=training_args, processing_class=tokenizer, train_dataset=train_dataset, eval_dataset=eval_dataset, data_collator=collate_fn, - **kwargs + **kwargs, ) self.loss_fn = MSELoss() - if args.checkpoint_path: - self.load_lora_checkpoint(args.checkpoint_path) + self.model.base_model.model.score.weight.register_hook( + lambda grad: print( + f"[HOOK] Gradient norm for score.weight: {grad.norm().item()}" + ) + ) + self.model.base_model.model.model.layers[ + 3 + ].self_attn.v_proj.lora_B.default.weight.register_hook( + lambda grad: print( + f"[HOOK] base_model.model.model.layers.3.self_attn.v_proj.lora_B.default.weight: {grad.norm().item()}" + ) + ) + for name, param in self.model.named_parameters(): + if param.requires_grad: + print(f"{name} shape={param.shape}") def setup_dataset(self, tokenizer): - env = Environment(loader=FileSystemLoader("/".join(self.args.template_path.split("/")[:-1]))) + env = Environment( + loader=FileSystemLoader("/".join(self.args.template_path.split("/")[:-1])) + ) template = env.get_template(self.args.template_path.split("/")[-1]) - dataset = RMDataset(self.args.reward_data_path, tokenizer, template, self.args.max_length) + dataset = RMDataset( + self.args.reward_data_path, tokenizer, template, self.args.max_length + ) if self.accelerator.is_main_process: print(f"dataset: {len(dataset)}") @@ -107,7 +147,9 @@ def setup_dataset(self, tokenizer): # Use deterministic splitter with seed to ensure same split across processes generator = torch.Generator().manual_seed(42) - train_dataset, val_dataset = random_split(dataset, [train_size, val_size], generator=generator) + train_dataset, val_dataset = random_split( + dataset, [train_size, val_size], generator=generator + ) return train_dataset, val_dataset def compute_loss(self, model, inputs, return_outputs=False, **kwargs): @@ -117,21 +159,40 @@ def compute_loss(self, model, inputs, return_outputs=False, **kwargs): outputs = model(input_ids, attention_mask=attention_masks) predicted_rewards = outputs.logits.squeeze(-1) # Shape: (batch_size,) + print(">>> Predicted:", predicted_rewards.detach().cpu().numpy()) + print(">>> GroundTruth:", true_rewards.detach().cpu().numpy()) loss = self.loss_fn(predicted_rewards, true_rewards) return (loss, outputs) if return_outputs else loss - def save_lora_checkpoint(self, output_dir=None, **kwargs): - if self.accelerator.is_main_process: - self.model.save_pretrained(output_dir) - - def load_lora_checkpoint(self, checkpoint_path): - adapter_model_path = os.path.join(checkpoint_path, 'adapter_model.safetensors') - peft_config = LoraConfig.from_pretrained(checkpoint_path) - - if os.path.exists(adapter_model_path): - self.model.load_adapter(checkpoint_path, adapter_name='lora', peft_config=peft_config) - else: - if self.accelerator.is_main_process: - print(f"No adapter model found at {adapter_model_path}.") - + def save_model( + self, output_dir: Optional[str] = None, _internal_call: bool = False + ): + """ + Override the default save_model to save merged full model (LoRA merged + score.weight). + """ + print("[Custom save_model] Called.") + if output_dir is None: + output_dir = self.args.output_dir + + if not self.accelerator.is_main_process: + return + print("[Custom save_model] Saving model...") + + os.makedirs(output_dir, exist_ok=True) + + try: + from copy import deepcopy + + param_check = self.model.base_model.model.score.weight + print( + f"mean={param_check.data.mean():.4f}, std={param_check.data.std():.4f}" + ) + model_copy = deepcopy(self.model) + merged_model = model_copy.merge_and_unload() + merged_model.save_pretrained(output_dir) + if hasattr(self, "tokenizer"): + self.tokenizer.save_pretrained(output_dir) + print(f"[Custom save_model] Full model saved to: {output_dir}") + except Exception as e: + print(f"[Custom save_model] Failed to merge and save full model: {str(e)}") From 26d06cc7390417a98585add380adf85d4a407922 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 19 Apr 2025 02:51:39 -0500 Subject: [PATCH 02/26] update --- scripts/inference_rm.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/scripts/inference_rm.py b/scripts/inference_rm.py index 4bcbb3d..c40dd87 100644 --- a/scripts/inference_rm.py +++ b/scripts/inference_rm.py @@ -39,26 +39,12 @@ def load_model_and_tokenizer(args): print("Using full precision model") model = AutoModelForSequenceClassification.from_pretrained( args.adapter_path, - # torch_dtype=torch.float32, #important - torch_dtype="auto", + torch_dtype=torch.float32, # important device_map="auto", num_labels=1, # For regression task pad_token_id=tokenizer.pad_token_id, # very important to add this ) - # adapter_path = os.path.join(args.adapter_path, 'adapter_model') - # # model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) - # if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): - # print(f"Loading adapter from: {args.adapter_path}") - # model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) - # else: - # print(f"No adapter found at {adapter_path}, using base model") - # model = base_model - # print("Active adapters:", model.active_adapters) - # score_weights = model.base_model.model.score.weight.data.view(-1) - # print("Mean of score.weight:", score_weights.mean().item()) - # print("Std of score.weight:", score_weights.std().item()) - # print("Model weight:", model.score.weight) def print_named_parameters(model, keyword="score"): for name, param in model.named_parameters(): if keyword in name: From 8687de89da30f69bbbfb964ad51d4542e8d75d19 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 19 Apr 2025 17:38:41 +0000 Subject: [PATCH 03/26] update --- scripts/accelerate_config_rm.yaml | 2 +- scripts/inference_rm.py | 2 +- scripts/inference_rm.sh | 2 +- scripts/train_rm.sh | 2 +- sotopia_rl/rm_trainer.py | 78 ++++--------------------------- 5 files changed, 13 insertions(+), 73 deletions(-) diff --git a/scripts/accelerate_config_rm.yaml b/scripts/accelerate_config_rm.yaml index ac91a13..fb193c9 100644 --- a/scripts/accelerate_config_rm.yaml +++ b/scripts/accelerate_config_rm.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 6 +num_processes: 5 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/inference_rm.py b/scripts/inference_rm.py index c40dd87..d23d198 100644 --- a/scripts/inference_rm.py +++ b/scripts/inference_rm.py @@ -39,7 +39,6 @@ def load_model_and_tokenizer(args): print("Using full precision model") model = AutoModelForSequenceClassification.from_pretrained( args.adapter_path, - torch_dtype=torch.float32, # important device_map="auto", num_labels=1, # For regression task pad_token_id=tokenizer.pad_token_id, # very important to add this @@ -77,6 +76,7 @@ def evaluate_prompt(model, tokenizer, prompt, index=None): print(f"\n[DEBUG] Prompt [{index}]:") print(prompt) print("[DEBUG] Decoded Input IDs:") + prompt = prompt.strip() encoded = tokenizer(prompt, return_tensors="pt", truncation=True) print(tokenizer.decode(encoded["input_ids"][0], skip_special_tokens=False)) diff --git a/scripts/inference_rm.sh b/scripts/inference_rm.sh index 376a53e..29ec65a 100644 --- a/scripts/inference_rm.sh +++ b/scripts/inference_rm.sh @@ -10,6 +10,6 @@ CUDA_VISIBLE_DEVICES=9 python inference_rm.py \ CUDA_VISIBLE_DEVICES=8 python inference_rm.py \ --model_path "/mnt/data_from_server1/models/Qwen2.5-7B-Instruct" \ - --adapter_path "/data/haofeiy2/sotopia-rl/rm_direct_overfit/checkpoint-200" \ + --adapter_path "/data/haofeiy2/sotopia-rl/rm_overfit_test/checkpoint-300" \ --template_path "/data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja" \ --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json" diff --git a/scripts/train_rm.sh b/scripts/train_rm.sh index 44b381d..df6b046 100644 --- a/scripts/train_rm.sh +++ b/scripts/train_rm.sh @@ -1,4 +1,4 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 accelerate launch \ +CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \ --main_process_port 29500 \ /data/haofeiy2/sotopia-rl/scripts/train_rm.py \ diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index c221884..0f11915 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -35,14 +35,14 @@ def __init__(self, args, accelerator, **kwargs): train_dataset, eval_dataset = self.setup_dataset(tokenizer) # Initialize wandb only on the main process - # if self.accelerator.is_main_process: - wandb.init( - project=args.wandb_project, - name=args.wandb_run_name, - config={ - k: v for k, v in vars(args).items() if isinstance(v, (int, float, str)) - }, - ) + if self.accelerator.is_main_process: + wandb.init( + project=args.wandb_project, + name=args.wandb_run_name, + config={ + k: v for k, v in vars(args).items() if isinstance(v, (int, float, str)) + }, + ) peft_config = LoraConfig( r=args.lora_r, @@ -59,20 +59,7 @@ def __init__(self, args, accelerator, **kwargs): # self.model = get_peft_model(base_model, peft_config) self.model = PeftModelForSequenceClassification(base_model, peft_config) - param_check = self.model.base_model.model.score.weight - print( - f"whether score.weight is trainable: {param_check.requires_grad}, shape: {param_check.shape}" - ) - print(f"mean={param_check.data.mean():.4f}, std={param_check.data.std():.4f}") - count = 0 - for name, param in self.model.named_parameters(): - if param.requires_grad: - count += 1 - print(f"{name} shape={param.shape}") - print(f"Total trainable parameters: {count}") - self.model.config.pad_token_id = ( - tokenizer.pad_token_id - ) # important to set the config pad_token_id + self.model.config.pad_token_id = tokenizer.pad_token_id # Set up the TrainingArguments with DeepSpeed support training_args = TrainingArguments( @@ -113,19 +100,6 @@ def __init__(self, args, accelerator, **kwargs): **kwargs, ) self.loss_fn = MSELoss() - - self.model.base_model.model.score.weight.register_hook( - lambda grad: print( - f"[HOOK] Gradient norm for score.weight: {grad.norm().item()}" - ) - ) - self.model.base_model.model.model.layers[ - 3 - ].self_attn.v_proj.lora_B.default.weight.register_hook( - lambda grad: print( - f"[HOOK] base_model.model.model.layers.3.self_attn.v_proj.lora_B.default.weight: {grad.norm().item()}" - ) - ) for name, param in self.model.named_parameters(): if param.requires_grad: print(f"{name} shape={param.shape}") @@ -159,40 +133,6 @@ def compute_loss(self, model, inputs, return_outputs=False, **kwargs): outputs = model(input_ids, attention_mask=attention_masks) predicted_rewards = outputs.logits.squeeze(-1) # Shape: (batch_size,) - print(">>> Predicted:", predicted_rewards.detach().cpu().numpy()) - print(">>> GroundTruth:", true_rewards.detach().cpu().numpy()) loss = self.loss_fn(predicted_rewards, true_rewards) return (loss, outputs) if return_outputs else loss - - def save_model( - self, output_dir: Optional[str] = None, _internal_call: bool = False - ): - """ - Override the default save_model to save merged full model (LoRA merged + score.weight). - """ - print("[Custom save_model] Called.") - if output_dir is None: - output_dir = self.args.output_dir - - if not self.accelerator.is_main_process: - return - print("[Custom save_model] Saving model...") - - os.makedirs(output_dir, exist_ok=True) - - try: - from copy import deepcopy - - param_check = self.model.base_model.model.score.weight - print( - f"mean={param_check.data.mean():.4f}, std={param_check.data.std():.4f}" - ) - model_copy = deepcopy(self.model) - merged_model = model_copy.merge_and_unload() - merged_model.save_pretrained(output_dir) - if hasattr(self, "tokenizer"): - self.tokenizer.save_pretrained(output_dir) - print(f"[Custom save_model] Full model saved to: {output_dir}") - except Exception as e: - print(f"[Custom save_model] Failed to merge and save full model: {str(e)}") From 7227da87a78ba4ca740d7bdfb71b78d153a6861c Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 19 Apr 2025 18:20:18 +0000 Subject: [PATCH 04/26] update --- scripts/inference_rm.py | 91 +++++++++++++--------------------------- scripts/inference_rm.sh | 2 +- scripts/train_rm.sh | 16 +++++++ sotopia_rl/rm_trainer.py | 4 -- 4 files changed, 46 insertions(+), 67 deletions(-) diff --git a/scripts/inference_rm.py b/scripts/inference_rm.py index d23d198..70a6e2c 100644 --- a/scripts/inference_rm.py +++ b/scripts/inference_rm.py @@ -6,60 +6,43 @@ from jinja2 import Environment, FileSystemLoader from peft import PeftModelForSequenceClassification from transformers import AutoModelForSequenceClassification, AutoTokenizer +from safetensors.torch import load_file +from safetensors import safe_open def parse_args(): parser = argparse.ArgumentParser( description="Test a reward model with a template and example data" ) - parser.add_argument( - "--model_path", - type=str, - required=True, - help="Path to base model or HF model name", - ) - parser.add_argument( - "--adapter_path", - type=str, - required=True, - help="Path to saved checkpoint directory", - ) - parser.add_argument( - "--template_path", type=str, required=True, help="Path to Jinja template file" - ) - parser.add_argument( - "--example_path", type=str, required=True, help="Path to example data JSON" - ) + parser.add_argument("--model_path", type=str, required=True, help="Path to base model or HF model name") + parser.add_argument("--adapter_path", type=str, required=True, help="Path to saved checkpoint directory") + parser.add_argument("--template_path", type=str, required=True, help="Path to Jinja template file") + parser.add_argument("--example_path", type=str, required=True, help="Path to example data JSON") return parser.parse_args() - def load_model_and_tokenizer(args): print(f"Loading base model: {args.model_path}") tokenizer = AutoTokenizer.from_pretrained(args.model_path) print("Using full precision model") - model = AutoModelForSequenceClassification.from_pretrained( - args.adapter_path, + base_model = AutoModelForSequenceClassification.from_pretrained( + args.model_path, device_map="auto", num_labels=1, # For regression task - pad_token_id=tokenizer.pad_token_id, # very important to add this + pad_token_id=tokenizer.pad_token_id # very important to add this ) - def print_named_parameters(model, keyword="score"): - for name, param in model.named_parameters(): - if keyword in name: - print( - f"{name}: mean={param.data.mean():.4f}, std={param.data.std():.4f}" - ) - else: - print("did not load score_weights") - print_named_parameters(model, keyword="score") + adapter_path = os.path.join(args.adapter_path, 'adapter_model') + if os.path.exists(adapter_path + '.safetensors') or os.path.exists(adapter_path + '.bin'): + print(f"Loading adapter from: {args.adapter_path}") + model = PeftModelForSequenceClassification.from_pretrained(base_model, args.adapter_path) + else: + print(f"No adapter found at {adapter_path}, using base model") + model = base_model model.eval() - return model, tokenizer - def load_template(template_path): template_dir = os.path.dirname(template_path) template_file = os.path.basename(template_path) @@ -68,43 +51,29 @@ def load_template(template_path): template_dir = "." env = Environment(loader=FileSystemLoader(template_dir)) - env.filters["tojson"] = lambda obj: json.dumps(obj) + env.filters['tojson'] = lambda obj: json.dumps(obj) return env.get_template(template_file) - -def evaluate_prompt(model, tokenizer, prompt, index=None): - print(f"\n[DEBUG] Prompt [{index}]:") - print(prompt) - print("[DEBUG] Decoded Input IDs:") - prompt = prompt.strip() - encoded = tokenizer(prompt, return_tensors="pt", truncation=True) - print(tokenizer.decode(encoded["input_ids"][0], skip_special_tokens=False)) - - # Check input length - print(f"[DEBUG] Input length: {encoded['input_ids'].shape[-1]} tokens") - # inputs = tokenizer(prompt, return_tensors="pt", truncation=True) +def evaluate_prompt(model, tokenizer, prompt): + inputs = tokenizer(prompt, return_tensors="pt", truncation=True) device = next(model.parameters()).device - inputs = {k: v.to(device) for k, v in encoded.items()} + inputs = {k: v.to(device) for k, v in inputs.items()} with torch.no_grad(): outputs = model(**inputs) # Get reward score directly from the logits - # reward = outputs.logits.squeeze().cpu().item() - logits = outputs.logits.squeeze() - print(f"[DEBUG] Raw logits: {logits}") - reward = logits.cpu().item() + reward = outputs.logits.squeeze().cpu().item() return reward - def main(): args = parse_args() model, tokenizer = load_model_and_tokenizer(args) - with open(args.example_path, "r") as f: + with open(args.example_path, 'r') as f: example_data = json.load(f) template = load_template(args.template_path) @@ -113,15 +82,14 @@ def main(): rendered_prompt = template.render( messages=[ - {"role": "user", "content": example["input"]}, - {"role": "assistant", "content": example["output"]}, + {"role": "user", "content": example['input']}, + {"role": "assistant", "content": example['output']}, ], - add_generation_prompt=False, - ) + add_generation_prompt=False + ).strip() - # reward = evaluate_prompt(model, tokenizer, rendered_prompt) - reward = evaluate_prompt(model, tokenizer, rendered_prompt, index=i + 1) - gth_reward = example.get("value") + reward = evaluate_prompt(model, tokenizer, rendered_prompt) + gth_reward = example.get('value') print(f"REWARD SCORE: {reward:.6f}") if gth_reward is not None: @@ -129,6 +97,5 @@ def main(): else: print("GTH REWARD: Not available") - if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/scripts/inference_rm.sh b/scripts/inference_rm.sh index 29ec65a..2218816 100644 --- a/scripts/inference_rm.sh +++ b/scripts/inference_rm.sh @@ -10,6 +10,6 @@ CUDA_VISIBLE_DEVICES=9 python inference_rm.py \ CUDA_VISIBLE_DEVICES=8 python inference_rm.py \ --model_path "/mnt/data_from_server1/models/Qwen2.5-7B-Instruct" \ - --adapter_path "/data/haofeiy2/sotopia-rl/rm_overfit_test/checkpoint-300" \ + --adapter_path "/data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-300" \ --template_path "/data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja" \ --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json" diff --git a/scripts/train_rm.sh b/scripts/train_rm.sh index df6b046..63f572a 100644 --- a/scripts/train_rm.sh +++ b/scripts/train_rm.sh @@ -13,3 +13,19 @@ CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ --reward_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --checkpoint_dir /data/haofeiy2/sotopia-rl/rm_token_length + +CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ + --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \ + --main_process_port 29500 \ + /data/haofeiy2/sotopia-rl/scripts/train_rm.py \ + --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ + --learning_rate 1e-5 \ + --max_length 4096 \ + --train_batch_size 1 \ + --val_batch_size 1 \ + --accumulation_steps 8 \ + --num_epochs 3000 \ + --evaluation_steps 50 \ + --reward_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json \ + --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ + --checkpoint_dir /data/haofeiy2/sotopia-rl/rm_overfit_test diff --git a/sotopia_rl/rm_trainer.py b/sotopia_rl/rm_trainer.py index 0f11915..b168ae5 100644 --- a/sotopia_rl/rm_trainer.py +++ b/sotopia_rl/rm_trainer.py @@ -82,7 +82,6 @@ def __init__(self, args, accelerator, **kwargs): ddp_find_unused_parameters=False, ) - print("Trainable parameters:") collate_fn = ( train_dataset.dataset.collate_fn @@ -100,9 +99,6 @@ def __init__(self, args, accelerator, **kwargs): **kwargs, ) self.loss_fn = MSELoss() - for name, param in self.model.named_parameters(): - if param.requires_grad: - print(f"{name} shape={param.shape}") def setup_dataset(self, tokenizer): env = Environment( From 52e0333766e2f8c79213ff2571211f398f477137 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 19 Apr 2025 19:00:03 +0000 Subject: [PATCH 05/26] update --- scripts/inference_rm.sh | 2 +- scripts/train_rm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/inference_rm.sh b/scripts/inference_rm.sh index 2218816..f000421 100644 --- a/scripts/inference_rm.sh +++ b/scripts/inference_rm.sh @@ -10,6 +10,6 @@ CUDA_VISIBLE_DEVICES=9 python inference_rm.py \ CUDA_VISIBLE_DEVICES=8 python inference_rm.py \ --model_path "/mnt/data_from_server1/models/Qwen2.5-7B-Instruct" \ - --adapter_path "/data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-300" \ + --adapter_path "/data/haofeiy2/sotopia-rl/rm_overfit_test/checkpoint-100" \ --template_path "/data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja" \ --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json" diff --git a/scripts/train_rm.sh b/scripts/train_rm.sh index 63f572a..d934b76 100644 --- a/scripts/train_rm.sh +++ b/scripts/train_rm.sh @@ -9,7 +9,7 @@ CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ --val_batch_size 1 \ --accumulation_steps 8 \ --num_epochs 30 \ - --evaluation_steps 100 \ + --evaluation_steps 50 \ --reward_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --checkpoint_dir /data/haofeiy2/sotopia-rl/rm_token_length From 693d90cff81676f9eef83da72aaa10bc7a39faf9 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sat, 19 Apr 2025 19:01:06 +0000 Subject: [PATCH 06/26] update --- scripts/inference_rm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/inference_rm.py b/scripts/inference_rm.py index 70a6e2c..09e81dd 100644 --- a/scripts/inference_rm.py +++ b/scripts/inference_rm.py @@ -6,8 +6,6 @@ from jinja2 import Environment, FileSystemLoader from peft import PeftModelForSequenceClassification from transformers import AutoModelForSequenceClassification, AutoTokenizer -from safetensors.torch import load_file -from safetensors import safe_open def parse_args(): From dfccbe635847b63831240b9febd5af6c686c89d6 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 08:33:18 +0000 Subject: [PATCH 07/26] update --- scripts/accelerate_config_ppo.yaml | 2 +- scripts/inference_rm.sh | 4 ++-- scripts/train_ppo.py | 2 +- scripts/train_ppo.sh | 23 +++++++++++------------ sotopia_rl/ppo_trainer.py | 11 +---------- 5 files changed, 16 insertions(+), 26 deletions(-) diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index d4c8212..cf0d84e 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -13,7 +13,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 6 +num_processes: 1 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/inference_rm.sh b/scripts/inference_rm.sh index f000421..18f8846 100644 --- a/scripts/inference_rm.sh +++ b/scripts/inference_rm.sh @@ -8,8 +8,8 @@ CUDA_VISIBLE_DEVICES=9 python inference_rm.py \ --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json" -CUDA_VISIBLE_DEVICES=8 python inference_rm.py \ +CUDA_VISIBLE_DEVICES=5 python inference_rm.py \ --model_path "/mnt/data_from_server1/models/Qwen2.5-7B-Instruct" \ - --adapter_path "/data/haofeiy2/sotopia-rl/rm_overfit_test/checkpoint-100" \ + --adapter_path "/data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800" \ --template_path "/data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja" \ --example_path "/data/haofeiy2/sotopia-rl/data/sotopia_pi_gpt4_rm_overfit.json" diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 7af4b70..02d2069 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -20,7 +20,7 @@ help="Number of PPO epochs per update") parser.add_argument("--learning_rate", type=float, default=5e-6, help="Learning rate for optimizer") - parser.add_argument("--gamma", type=float, default=0.99, + parser.add_argument("--gamma", type=float, default=1.0, help="Discount factor") parser.add_argument("--lam", type=float, default=0.95, help="GAE lambda for advantage estimation") diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index b9d8f5e..ebe24f9 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,21 +1,20 @@ -CUDA_VISIBLE_DEVICES=1,7,8 accelerate launch \ - --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config.yaml \ +CUDA_VISIBLE_DEVICES=5 accelerate launch \ + --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29511 \ - /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ + /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ - --value_adapter_path /data/haofeiy2/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ - --learning_rate 1e-5 \ + --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800 \ + --value_adapter_path /data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800 \ + --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 5 \ --num_mini_batches 1 \ --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ - --num_ppo_epochs 2 \ --num_train_epochs 5 \ - --gamma 0.99 \ - --lam 0.95 \ - --output_dir /data/haofeiy2/sotopia-rl/ppo_origin_qwen25_7b_reward_direct_default_no_goal_gpt-4o_without_goal_leak_with_sft_self_play_data_use_sotopia_pi_full_data_0408 \ No newline at end of file + --max_length 4096 \ + --use_lora_train_ppo \ + --output_dir /data/haofeiy2/sotopia-rl/ppo_token_length \ No newline at end of file diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index bde99fc..71c6586 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -31,15 +31,6 @@ def __init__(self, args, accelerator: Accelerator): self._setup_generation_models() self._setup_classification_models() - - self.policy, self.ref_policy, self.reward_model, self.value_model = self.accelerator.prepare( - self.policy, self.ref_policy, self.reward_model, self.value_model - ) - self.policy = self.accelerator.unwrap_model(self.policy) - self.ref_policy = self.accelerator.unwrap_model(self.ref_policy) - self.reward_model = self.accelerator.unwrap_model(self.reward_model) - self.value_model = self.accelerator.unwrap_model(self.value_model) - self._setup_ppo_trainer() def save_model(self, output_dir: str, _internal_call: bool = False): @@ -84,7 +75,7 @@ def _setup_dataset(self): generator = torch.Generator().manual_seed(42) val_ratio = getattr(self.args, 'val_ratio', 0.05) - train_size = min(int(len(dataset) * (1 - val_ratio)), len(dataset) - 2) + train_size = min(int(len(dataset) * (1 - val_ratio)), len(dataset) - 10) val_size = len(dataset) - train_size self.train_dataset, self.val_dataset = random_split(dataset, [train_size, val_size], generator=generator) print(f"Dataset split: {len(self.train_dataset)} train, {len(self.val_dataset)} validation") From fcae8dd550fb733f2f1787873101191cce963db7 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 08:38:16 +0000 Subject: [PATCH 08/26] update --- scripts/train_ppo.py | 3 +-- scripts/train_ppo.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 02d2069..a8bdfec 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -69,6 +69,5 @@ help="Use LoRA for training PPO") args = parser.parse_args() - accelerator = Accelerator() - trainer = SotopiaPPOTrainer(args, accelerator) + trainer = SotopiaPPOTrainer(args) trainer.train() diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index ebe24f9..5aee9e0 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -10,7 +10,7 @@ CUDA_VISIBLE_DEVICES=5 accelerate launch \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ - --gradient_accumulation_steps 5 \ + --gradient_accumulation_steps 1 \ --num_mini_batches 1 \ --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ From 67d8ff077d82a6c30252740fd6b72d811a111baf Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 08:39:55 +0000 Subject: [PATCH 09/26] update --- sotopia_rl/ppo_trainer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 71c6586..b72576e 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -20,8 +20,7 @@ os.environ["TORCH_DISTRIBUTED_DEBUG"] = "DETAIL" class SotopiaPPOTrainer: - def __init__(self, args, accelerator: Accelerator): - self.accelerator = accelerator + def __init__(self, args): self.args = args self._init_wandb() From acc794b4bc371ae2b0287d1cdaee02d52aaf6893 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 09:31:23 +0000 Subject: [PATCH 10/26] disable dropout --- sotopia_rl/ppo_trainer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index b72576e..3aa5b9d 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -12,6 +12,7 @@ GenerationConfig, ) from trl import get_kbit_device_map, PPOConfig, PPOTrainer +from trl.trainer.utils import disable_dropout_in_model from accelerate import PartialState, Accelerator import wandb @@ -100,6 +101,7 @@ def _setup_generation_models(self): is_trainable=False, adapter_name="ref_adapter" ) + disable_dropout_in_model(self.ref_policy) if self.args.use_lora_train_ppo: base_gen_policy = AutoModelForCausalLM.from_pretrained( @@ -119,6 +121,7 @@ def _setup_generation_models(self): self.args.model_name, torch_dtype='auto', ) + disable_dropout_in_model(self.policy) requires_grad_num = 0 for name, param in self.policy.named_parameters(): @@ -147,6 +150,8 @@ def _setup_classification_models(self): is_trainable=False, adapter_name="reward_adapter" ) + disable_dropout_in_model(self.reward_model) + for name, param in self.reward_model.named_parameters(): if self.reward_model.active_adapter in name: param.requires_grad = False @@ -171,6 +176,7 @@ def _setup_classification_models(self): torch_dtype='auto', num_labels=1, ) + disable_dropout_in_model(self.value_model) # VERY VERY IMPORTANT # specifically designed for PPO training, From b24cfa802ef97321081339e1c422f78e2597e2c7 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 10:57:37 +0000 Subject: [PATCH 11/26] update --- scripts/train_ppo.sh | 4 ++-- sotopia_rl/ppo_trainer.py | 5 +++++ sotopia_rl/sft_trainer.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 5aee9e0..9d84694 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,6 +1,6 @@ CUDA_VISIBLE_DEVICES=5 accelerate launch \ --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29511 \ + --main_process_port 29519 \ /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ @@ -11,7 +11,7 @@ CUDA_VISIBLE_DEVICES=5 accelerate launch \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 1 \ - --num_mini_batches 1 \ + --num_mini_batches 4 \ --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 3aa5b9d..3aefd8e 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -12,6 +12,7 @@ GenerationConfig, ) from trl import get_kbit_device_map, PPOConfig, PPOTrainer +from peft import prepare_model_for_kbit_training from trl.trainer.utils import disable_dropout_in_model from accelerate import PartialState, Accelerator @@ -95,6 +96,7 @@ def _setup_generation_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_gen_ref = prepare_model_for_kbit_training(base_gen_ref) self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_ref, self.args.ref_adapter_path, @@ -110,6 +112,7 @@ def _setup_generation_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_gen_policy = prepare_model_for_kbit_training(base_gen_policy) self.policy = PeftModelForCausalLM.from_pretrained( base_gen_policy, self.args.policy_adapter_path, @@ -144,6 +147,7 @@ def _setup_classification_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_reward_model = prepare_model_for_kbit_training(base_reward_model) self.reward_model = PeftModelForSequenceClassification.from_pretrained( base_reward_model, self.args.reward_adapter_path, @@ -164,6 +168,7 @@ def _setup_classification_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_value_model = prepare_model_for_kbit_training(base_value_model) # important for qlora self.value_model = PeftModelForSequenceClassification.from_pretrained( base_value_model, self.args.value_adapter_path, diff --git a/sotopia_rl/sft_trainer.py b/sotopia_rl/sft_trainer.py index c0a87c7..480b0ca 100644 --- a/sotopia_rl/sft_trainer.py +++ b/sotopia_rl/sft_trainer.py @@ -51,6 +51,7 @@ def __init__(self, args, accelerator): torch_dtype=torch.float16, quantization_config=quantization_config, ) + base_model = prepare_model_for_kbit_training(base_model) else: base_model = AutoModelForCausalLM.from_pretrained(args.model_name).to(self.device) From a93af28c318c09439230a0da53cbf6df1967f7f1 Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 20:54:55 +0000 Subject: [PATCH 12/26] update --- sotopia_rl/ppo_trainer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 3aefd8e..8eaac53 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -22,7 +22,8 @@ os.environ["TORCH_DISTRIBUTED_DEBUG"] = "DETAIL" class SotopiaPPOTrainer: - def __init__(self, args): + def __init__(self, args, accelerator: Accelerator): + self.accelerator = accelerator self.args = args self._init_wandb() @@ -48,11 +49,12 @@ def save_model(self, output_dir: str, _internal_call: bool = False): self.ppo_trainer.save_model = save_model.__get__(self.ppo_trainer, type(self.ppo_trainer)) def _init_wandb(self): - wandb.init( - project=self.args.wandb_project, - name=self.args.wandb_run_name, - config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} - ) + if self.accelerator.is_main_process: + wandb.init( + project=self.args.wandb_project, + name=self.args.wandb_run_name, + config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} + ) def _setup_tokenizer(self): self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left") From 8f0d0946136707caffc2f108802fc7b9f091489a Mon Sep 17 00:00:00 2001 From: lwaekfjlk <1125027232@qq.com> Date: Sun, 20 Apr 2025 21:01:35 +0000 Subject: [PATCH 13/26] update --- evals/redis_stat.py | 2 +- scripts/accelerate_config_ppo.yaml | 7 +------ scripts/train_ppo.py | 3 ++- scripts/train_ppo.sh | 3 ++- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/evals/redis_stat.py b/evals/redis_stat.py index 3e08f53..423079a 100644 --- a/evals/redis_stat.py +++ b/evals/redis_stat.py @@ -129,4 +129,4 @@ def analyze_episodes_with_positions(tag): } # Run the analysis -results = analyze_episodes_with_positions("Qwen2.5-7B-Instruct_vs_sft_qwen25_7b_bigtom_step_1500-bigtom_0402") +results = analyze_episodes_with_positions("grpo_direct_step_400_vs_sft_qwen25_7b_sft_round_1_bc_data_top_2_step_1500-0420") diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index cf0d84e..4cd0426 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -2,18 +2,13 @@ compute_environment: LOCAL_MACHINE debug: true distributed_type: MULTI_GPU downcast_bf16: 'no' -dynamo_config: - dynamo_backend: EAGER - dynamo_mode: default - dynamo_use_dynamic: false - dynamo_use_fullgraph: false enable_cpu_affinity: true gpu_ids: all machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 1 +num_processes: 3 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index a8bdfec..02d2069 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -69,5 +69,6 @@ help="Use LoRA for training PPO") args = parser.parse_args() - trainer = SotopiaPPOTrainer(args) + accelerator = Accelerator() + trainer = SotopiaPPOTrainer(args, accelerator) trainer.train() diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 9d84694..bba6369 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,4 +1,4 @@ -CUDA_VISIBLE_DEVICES=5 accelerate launch \ +CUDA_VISIBLE_DEVICES=5,6,7 accelerate launch \ --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29519 \ /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ @@ -16,5 +16,6 @@ CUDA_VISIBLE_DEVICES=5 accelerate launch \ --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ + --num_ppo_epochs 1 \ --use_lora_train_ppo \ --output_dir /data/haofeiy2/sotopia-rl/ppo_token_length \ No newline at end of file From a6a9ba1d0d69117eac2d2b67de5d94fe3c2c9dc1 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Tue, 22 Apr 2025 14:31:03 +0800 Subject: [PATCH 14/26] update --- scripts/accelerate_config_ppo.yaml | 2 +- scripts/accelerate_config_rm.yaml | 2 +- scripts/train_ppo.py | 3 +- scripts/train_ppo.sh | 31 +++--- scripts/train_rm.sh | 18 ++-- sotopia_rl/grpo_trainer_math.py | 164 +++++++++++++++++++++++++++++ sotopia_rl/ppo_trainer.py | 61 ++++++----- 7 files changed, 222 insertions(+), 59 deletions(-) create mode 100644 sotopia_rl/grpo_trainer_math.py diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index 4cd0426..f025f6d 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 3 +num_processes: 8 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/accelerate_config_rm.yaml b/scripts/accelerate_config_rm.yaml index fb193c9..5b42349 100644 --- a/scripts/accelerate_config_rm.yaml +++ b/scripts/accelerate_config_rm.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 5 +num_processes: 8 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index 02d2069..a8bdfec 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -69,6 +69,5 @@ help="Use LoRA for training PPO") args = parser.parse_args() - accelerator = Accelerator() - trainer = SotopiaPPOTrainer(args, accelerator) + trainer = SotopiaPPOTrainer(args) trainer.train() diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index bba6369..d03b5c4 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,21 +1,22 @@ -CUDA_VISIBLE_DEVICES=5,6,7 accelerate launch \ - --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29519 \ - /data/haofeiy2/sotopia-rl/scripts/train_ppo.py \ - --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --ref_adapter_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800 \ - --value_adapter_path /data/haofeiy2/sotopia-rl/rm_token_length/checkpoint-800 \ - --learning_rate 1e-6 \ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29521 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-350 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-50 \ + --learning_rate 5e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 1 \ - --num_mini_batches 4 \ - --ppo_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ - --num_ppo_epochs 1 \ + --num_ppo_epochs 4 \ + --gamma 0.95 \ --use_lora_train_ppo \ - --output_dir /data/haofeiy2/sotopia-rl/ppo_token_length \ No newline at end of file + --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized diff --git a/scripts/train_rm.sh b/scripts/train_rm.sh index d934b76..1a8ae68 100644 --- a/scripts/train_rm.sh +++ b/scripts/train_rm.sh @@ -1,18 +1,18 @@ -CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ - --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_rm.yaml \ --main_process_port 29500 \ - /data/haofeiy2/sotopia-rl/scripts/train_rm.py \ - --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ + /data/disk0/sotopia-rl/scripts/train_rm.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --learning_rate 1e-5 \ --max_length 4096 \ - --train_batch_size 1 \ + --train_batch_size 4 \ --val_batch_size 1 \ - --accumulation_steps 8 \ + --accumulation_steps 2 \ --num_epochs 30 \ --evaluation_steps 50 \ - --reward_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length.json \ - --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ - --checkpoint_dir /data/haofeiy2/sotopia-rl/rm_token_length + --reward_data_path /data/disk0/sotopia-rl/data/sotopia_pi_bc_episodes_reward_token_length_binary.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --checkpoint_dir /data/disk0/sotopia-rl/rm_token_length_binary CUDA_VISIBLE_DEVICES=5,6,7,8,9 accelerate launch \ --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_rm.yaml \ diff --git a/sotopia_rl/grpo_trainer_math.py b/sotopia_rl/grpo_trainer_math.py new file mode 100644 index 0000000..d6f3a62 --- /dev/null +++ b/sotopia_rl/grpo_trainer_math.py @@ -0,0 +1,164 @@ +import os +import torch +import wandb +from datasets import load_dataset +from torch.utils.data import random_split +from transformers import ( + AutoModelForCausalLM, + AutoModelForSequenceClassification, + AutoTokenizer, + BitsAndBytesConfig, + GenerationConfig, +) +from accelerate import PartialState +from peft import PeftModelForCausalLM, PeftModelForSequenceClassification +from jinja2 import Environment, FileSystemLoader +from trl import get_kbit_device_map, GRPOConfig, GRPOTrainer +from accelerate import Accelerator +from sotopia_rl.data import GRPODataset +from functools import partial +from typing import List + +os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" +os.environ['NCCL_P2P_DISABLE'] = '1' +os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'DETAIL' + +SIMPLE_CHAT_TEMPLATE = "{% for message in messages %}{{message['role'].capitalize() + ': ' + message['content'] + '\n\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}" + + +from transformers import GPTNeoXForCausalLM + +class PatchedGPTNeoXForCausalLM(GPTNeoXForCausalLM): + def forward(self, *args, logits_to_keep=None, **kwargs): + return super().forward(*args, **kwargs) + +class SotopiaGRPOTrainer: + def __init__(self, args, accelerator: Accelerator): + self.args = args + self.accelerator = accelerator + + self._init_wandb() + self._setup_tokenizer() + self._setup_dataset() + self._create_quantization_config() + + self._setup_grpo_trainer() + + def save_model(self, output_dir: str, _internal_call: bool = False): + self.model.save_pretrained(output_dir) + self.tokenizer.save_pretrained(output_dir) + print(f"Saved PEFT model to {output_dir}") + + self.grpo_trainer.save_model = save_model.__get__(self.grpo_trainer, type(self.grpo_trainer)) + + def _init_wandb(self): + wandb.init( + project=self.args.wandb_project, + name=self.args.wandb_run_name, + config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} + ) + + def _setup_tokenizer(self): + self.tokenizer = AutoTokenizer.from_pretrained("/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr") + self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) + self.tokenizer.pad_token_id = self.tokenizer.convert_tokens_to_ids('[PAD]') + if self.tokenizer.chat_template is None: + self.tokenizer.chat_template = SIMPLE_CHAT_TEMPLATE + + + def _setup_dataset(self): + from datasets import load_dataset + + dataset = load_dataset("trl-internal-testing/tldr-preference-sft-trl-style") + print("processing") + train_dataset = dataset["train"] + eval_dataset = dataset["validation"] + + def prepare_dataset(dataset, tokenizer): + def tokenize(element): + input_ids = tokenizer.apply_chat_template( + element["messages"][:1], + padding=False, + add_generation_prompt=True, + ) + return {"input_ids": input_ids, "lengths": len(input_ids), "prompt": element["messages"][:1]} + + return dataset.map( + tokenize, + remove_columns=dataset.column_names, + num_proc=4, + ) + + with PartialState().local_main_process_first(): + train_dataset = prepare_dataset(train_dataset, self.tokenizer) + if eval_dataset is not None: + eval_dataset = prepare_dataset(eval_dataset, self.tokenizer) + train_dataset = train_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) + if eval_dataset is not None: + eval_dataset = eval_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) + + assert train_dataset[0]["input_ids"][-1] != self.tokenizer.eos_token_id, "The last token should not be an EOS token" + + self.train_dataset = train_dataset + self.val_dataset = eval_dataset + print(f"Dataset loaded and processed: {len(self.train_dataset)} train, {len(self.val_dataset or [])} validation") + + def _create_quantization_config(self): + self.quant_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.bfloat16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type="nf4" + ) + + def _setup_grpo_trainer(self): + num_processes = self.accelerator.num_processes + global_batch_size = self.args.per_device_train_batch_size * num_processes + + num_generations = 4 # manually chosen value + print(f"Using num_generations = {num_generations} (global_batch_size = {global_batch_size})") + + policy_model = AutoModelForCausalLM.from_pretrained( + "/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr", + torch_dtype='auto', + num_labels=1, + ) + + reward_model = AutoModelForSequenceClassification.from_pretrained( + "/data/disk0/models/EleutherAI_pythia-1b-deduped__reward__tldr", + torch_dtype='auto', + num_labels=1, + ) + + training_args = GRPOConfig( + logging_steps = 1, + report_to = "wandb", + per_device_train_batch_size=self.args.per_device_train_batch_size, + per_device_eval_batch_size=self.args.per_device_eval_batch_size, + gradient_accumulation_steps=self.args.gradient_accumulation_steps, + num_train_epochs=self.args.num_train_epochs, + learning_rate=self.args.learning_rate, + output_dir=self.args.output_dir, + save_steps=self.args.save_steps, + num_generations=num_generations + ) + + self.grpo_trainer = GRPOTrainer( + args=training_args, + model=policy_model, + reward_funcs=reward_model, + processing_class=self.tokenizer, + reward_processing_classes=self.tokenizer, + train_dataset=self.train_dataset, + eval_dataset=self.val_dataset, + ) + print("GRPOtrainer setup complete") + + def train(self): + try: + print("Starting GRPO training...") + train_stats = self.grpo_trainer.train() + return train_stats + except Exception as e: + print(f"Training error: {str(e)}") + raise diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 8eaac53..c7fe603 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -22,8 +22,7 @@ os.environ["TORCH_DISTRIBUTED_DEBUG"] = "DETAIL" class SotopiaPPOTrainer: - def __init__(self, args, accelerator: Accelerator): - self.accelerator = accelerator + def __init__(self, args): self.args = args self._init_wandb() @@ -35,26 +34,33 @@ def __init__(self, args, accelerator: Accelerator): self._setup_classification_models() self._setup_ppo_trainer() + for m in [self.policy, self.ref_policy]: + m.config.use_cache = False + for m in [self.value_model, self.reward_model]: + m.config.use_cache = False + def save_model(self, output_dir: str, _internal_call: bool = False): if hasattr(self.model, "policy"): - model_to_save = self.model.policy - elif hasattr(self.model, "module") and hasattr(self.model.module, "policy"): - model_to_save = self.model.module.policy + policy_to_save = self.model.policy + value_to_save = self.model.value_model + elif hasattr(self.model, "module"): + policy_to_save = self.model.module.policy + value_to_save = self.model.module.value_model else: - model_to_save = self.model - model_to_save.save_pretrained(output_dir) + raise ValueError("Model does not have 'policy' or 'module' attribute") + policy_to_save.save_pretrained(output_dir) + value_to_save.save_pretrained(output_dir) self.tokenizer.save_pretrained(output_dir) print(f"Model saved to {output_dir}") self.ppo_trainer.save_model = save_model.__get__(self.ppo_trainer, type(self.ppo_trainer)) def _init_wandb(self): - if self.accelerator.is_main_process: - wandb.init( - project=self.args.wandb_project, - name=self.args.wandb_run_name, - config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} - ) + wandb.init( + project=self.args.wandb_project, + name=self.args.wandb_run_name, + config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} + ) def _setup_tokenizer(self): self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left") @@ -69,9 +75,9 @@ def _setup_dataset(self): # Create and split dataset dataset = PPODataset( - self.args.ppo_data_path, - self.tokenizer, - template, + data_path=self.args.ppo_data_path, + tokenizer=self.tokenizer, + template=template, max_length=self.args.max_length ) print(f"dataset: {len(dataset)}") @@ -98,15 +104,14 @@ def _setup_generation_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) - base_gen_ref = prepare_model_for_kbit_training(base_gen_ref) self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_ref, self.args.ref_adapter_path, is_trainable=False, adapter_name="ref_adapter" ) - disable_dropout_in_model(self.ref_policy) + if self.args.use_lora_train_ppo: base_gen_policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, @@ -114,7 +119,6 @@ def _setup_generation_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) - base_gen_policy = prepare_model_for_kbit_training(base_gen_policy) self.policy = PeftModelForCausalLM.from_pretrained( base_gen_policy, self.args.policy_adapter_path, @@ -126,7 +130,8 @@ def _setup_generation_models(self): self.args.model_name, torch_dtype='auto', ) - disable_dropout_in_model(self.policy) + self.ref_policy.config.pad_token_id = self.tokenizer.pad_token_id + self.policy.config.pad_token_id = self.tokenizer.pad_token_id requires_grad_num = 0 for name, param in self.policy.named_parameters(): @@ -149,14 +154,12 @@ def _setup_classification_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) - base_reward_model = prepare_model_for_kbit_training(base_reward_model) self.reward_model = PeftModelForSequenceClassification.from_pretrained( base_reward_model, self.args.reward_adapter_path, is_trainable=False, adapter_name="reward_adapter" ) - disable_dropout_in_model(self.reward_model) for name, param in self.reward_model.named_parameters(): if self.reward_model.active_adapter in name: @@ -170,7 +173,6 @@ def _setup_classification_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) - base_value_model = prepare_model_for_kbit_training(base_value_model) # important for qlora self.value_model = PeftModelForSequenceClassification.from_pretrained( base_value_model, self.args.value_adapter_path, @@ -183,14 +185,10 @@ def _setup_classification_models(self): torch_dtype='auto', num_labels=1, ) - disable_dropout_in_model(self.value_model) - # VERY VERY IMPORTANT - # specifically designed for PPO training, - # based on the get_reward function - # it fill the input_ids paddings with 0s - self.value_model.config.pad_token_id = 0 - self.reward_model.config.pad_token_id = 0 + # need to set this with not None results + self.value_model.config.pad_token_id = self.tokenizer.pad_token_id + self.reward_model.config.pad_token_id = self.tokenizer.pad_token_id requires_grad_num = 0 for name, param in self.value_model.named_parameters(): @@ -206,6 +204,7 @@ def _setup_classification_models(self): def _setup_ppo_trainer(self): training_args = PPOConfig( + vf_coef=0.5, per_device_train_batch_size=self.args.per_device_train_batch_size, per_device_eval_batch_size=self.args.per_device_eval_batch_size, num_mini_batches=self.args.num_mini_batches, @@ -243,4 +242,4 @@ def train(self): return train_stats except Exception as e: print(f"Training error: {str(e)}") - raise \ No newline at end of file + raise From f8b61dd85887660a1173d566e86f984a43b3f028 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Tue, 22 Apr 2025 14:34:03 +0800 Subject: [PATCH 15/26] update --- sotopia_rl/grpo_trainer_math.py | 164 -------------------------------- 1 file changed, 164 deletions(-) delete mode 100644 sotopia_rl/grpo_trainer_math.py diff --git a/sotopia_rl/grpo_trainer_math.py b/sotopia_rl/grpo_trainer_math.py deleted file mode 100644 index d6f3a62..0000000 --- a/sotopia_rl/grpo_trainer_math.py +++ /dev/null @@ -1,164 +0,0 @@ -import os -import torch -import wandb -from datasets import load_dataset -from torch.utils.data import random_split -from transformers import ( - AutoModelForCausalLM, - AutoModelForSequenceClassification, - AutoTokenizer, - BitsAndBytesConfig, - GenerationConfig, -) -from accelerate import PartialState -from peft import PeftModelForCausalLM, PeftModelForSequenceClassification -from jinja2 import Environment, FileSystemLoader -from trl import get_kbit_device_map, GRPOConfig, GRPOTrainer -from accelerate import Accelerator -from sotopia_rl.data import GRPODataset -from functools import partial -from typing import List - -os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" -os.environ['NCCL_P2P_DISABLE'] = '1' -os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'DETAIL' - -SIMPLE_CHAT_TEMPLATE = "{% for message in messages %}{{message['role'].capitalize() + ': ' + message['content'] + '\n\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}" - - -from transformers import GPTNeoXForCausalLM - -class PatchedGPTNeoXForCausalLM(GPTNeoXForCausalLM): - def forward(self, *args, logits_to_keep=None, **kwargs): - return super().forward(*args, **kwargs) - -class SotopiaGRPOTrainer: - def __init__(self, args, accelerator: Accelerator): - self.args = args - self.accelerator = accelerator - - self._init_wandb() - self._setup_tokenizer() - self._setup_dataset() - self._create_quantization_config() - - self._setup_grpo_trainer() - - def save_model(self, output_dir: str, _internal_call: bool = False): - self.model.save_pretrained(output_dir) - self.tokenizer.save_pretrained(output_dir) - print(f"Saved PEFT model to {output_dir}") - - self.grpo_trainer.save_model = save_model.__get__(self.grpo_trainer, type(self.grpo_trainer)) - - def _init_wandb(self): - wandb.init( - project=self.args.wandb_project, - name=self.args.wandb_run_name, - config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} - ) - - def _setup_tokenizer(self): - self.tokenizer = AutoTokenizer.from_pretrained("/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr") - self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) - self.tokenizer.pad_token_id = self.tokenizer.convert_tokens_to_ids('[PAD]') - if self.tokenizer.chat_template is None: - self.tokenizer.chat_template = SIMPLE_CHAT_TEMPLATE - - - def _setup_dataset(self): - from datasets import load_dataset - - dataset = load_dataset("trl-internal-testing/tldr-preference-sft-trl-style") - print("processing") - train_dataset = dataset["train"] - eval_dataset = dataset["validation"] - - def prepare_dataset(dataset, tokenizer): - def tokenize(element): - input_ids = tokenizer.apply_chat_template( - element["messages"][:1], - padding=False, - add_generation_prompt=True, - ) - return {"input_ids": input_ids, "lengths": len(input_ids), "prompt": element["messages"][:1]} - - return dataset.map( - tokenize, - remove_columns=dataset.column_names, - num_proc=4, - ) - - with PartialState().local_main_process_first(): - train_dataset = prepare_dataset(train_dataset, self.tokenizer) - if eval_dataset is not None: - eval_dataset = prepare_dataset(eval_dataset, self.tokenizer) - train_dataset = train_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) - if eval_dataset is not None: - eval_dataset = eval_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) - - assert train_dataset[0]["input_ids"][-1] != self.tokenizer.eos_token_id, "The last token should not be an EOS token" - - self.train_dataset = train_dataset - self.val_dataset = eval_dataset - print(f"Dataset loaded and processed: {len(self.train_dataset)} train, {len(self.val_dataset or [])} validation") - - def _create_quantization_config(self): - self.quant_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_compute_dtype=torch.bfloat16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4" - ) - - def _setup_grpo_trainer(self): - num_processes = self.accelerator.num_processes - global_batch_size = self.args.per_device_train_batch_size * num_processes - - num_generations = 4 # manually chosen value - print(f"Using num_generations = {num_generations} (global_batch_size = {global_batch_size})") - - policy_model = AutoModelForCausalLM.from_pretrained( - "/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr", - torch_dtype='auto', - num_labels=1, - ) - - reward_model = AutoModelForSequenceClassification.from_pretrained( - "/data/disk0/models/EleutherAI_pythia-1b-deduped__reward__tldr", - torch_dtype='auto', - num_labels=1, - ) - - training_args = GRPOConfig( - logging_steps = 1, - report_to = "wandb", - per_device_train_batch_size=self.args.per_device_train_batch_size, - per_device_eval_batch_size=self.args.per_device_eval_batch_size, - gradient_accumulation_steps=self.args.gradient_accumulation_steps, - num_train_epochs=self.args.num_train_epochs, - learning_rate=self.args.learning_rate, - output_dir=self.args.output_dir, - save_steps=self.args.save_steps, - num_generations=num_generations - ) - - self.grpo_trainer = GRPOTrainer( - args=training_args, - model=policy_model, - reward_funcs=reward_model, - processing_class=self.tokenizer, - reward_processing_classes=self.tokenizer, - train_dataset=self.train_dataset, - eval_dataset=self.val_dataset, - ) - print("GRPOtrainer setup complete") - - def train(self): - try: - print("Starting GRPO training...") - train_stats = self.grpo_trainer.train() - return train_stats - except Exception as e: - print(f"Training error: {str(e)}") - raise From 164cb5c9de2af3c457e15848f100b9d18ef4a947 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Tue, 22 Apr 2025 14:35:11 +0800 Subject: [PATCH 16/26] update --- sotopia_rl/sft_trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sotopia_rl/sft_trainer.py b/sotopia_rl/sft_trainer.py index 480b0ca..c0a87c7 100644 --- a/sotopia_rl/sft_trainer.py +++ b/sotopia_rl/sft_trainer.py @@ -51,7 +51,6 @@ def __init__(self, args, accelerator): torch_dtype=torch.float16, quantization_config=quantization_config, ) - base_model = prepare_model_for_kbit_training(base_model) else: base_model = AutoModelForCausalLM.from_pretrained(args.model_name).to(self.device) From 7307afd56d5b16479244b6df60276445c96e7ab4 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Tue, 22 Apr 2025 23:51:04 +0800 Subject: [PATCH 17/26] reward start increasing --- scripts/train_ppo.sh | 12 +-- sotopia_rl/grpo_trainer_math.py | 164 ++++++++++++++++++++++++++++++++ sotopia_rl/ppo_trainer.py | 11 ++- 3 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 sotopia_rl/grpo_trainer_math.py diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index d03b5c4..74f7fb9 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,14 +1,14 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29521 \ + --main_process_port 29529 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-350 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-50 \ - --learning_rate 5e-6 \ - --per_device_train_batch_size 1 \ + --value_adapter_path /data/disk0/sotopia-rl/ppo_token_length_normalized_checkpoint_510_value_adapter \ + --learning_rate 5e-5 \ + --per_device_train_batch_size 6 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 1 \ --num_mini_batches 1 \ @@ -16,7 +16,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ - --num_ppo_epochs 4 \ - --gamma 0.95 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ --use_lora_train_ppo \ --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized diff --git a/sotopia_rl/grpo_trainer_math.py b/sotopia_rl/grpo_trainer_math.py new file mode 100644 index 0000000..d6f3a62 --- /dev/null +++ b/sotopia_rl/grpo_trainer_math.py @@ -0,0 +1,164 @@ +import os +import torch +import wandb +from datasets import load_dataset +from torch.utils.data import random_split +from transformers import ( + AutoModelForCausalLM, + AutoModelForSequenceClassification, + AutoTokenizer, + BitsAndBytesConfig, + GenerationConfig, +) +from accelerate import PartialState +from peft import PeftModelForCausalLM, PeftModelForSequenceClassification +from jinja2 import Environment, FileSystemLoader +from trl import get_kbit_device_map, GRPOConfig, GRPOTrainer +from accelerate import Accelerator +from sotopia_rl.data import GRPODataset +from functools import partial +from typing import List + +os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" +os.environ['NCCL_P2P_DISABLE'] = '1' +os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'DETAIL' + +SIMPLE_CHAT_TEMPLATE = "{% for message in messages %}{{message['role'].capitalize() + ': ' + message['content'] + '\n\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}" + + +from transformers import GPTNeoXForCausalLM + +class PatchedGPTNeoXForCausalLM(GPTNeoXForCausalLM): + def forward(self, *args, logits_to_keep=None, **kwargs): + return super().forward(*args, **kwargs) + +class SotopiaGRPOTrainer: + def __init__(self, args, accelerator: Accelerator): + self.args = args + self.accelerator = accelerator + + self._init_wandb() + self._setup_tokenizer() + self._setup_dataset() + self._create_quantization_config() + + self._setup_grpo_trainer() + + def save_model(self, output_dir: str, _internal_call: bool = False): + self.model.save_pretrained(output_dir) + self.tokenizer.save_pretrained(output_dir) + print(f"Saved PEFT model to {output_dir}") + + self.grpo_trainer.save_model = save_model.__get__(self.grpo_trainer, type(self.grpo_trainer)) + + def _init_wandb(self): + wandb.init( + project=self.args.wandb_project, + name=self.args.wandb_run_name, + config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} + ) + + def _setup_tokenizer(self): + self.tokenizer = AutoTokenizer.from_pretrained("/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr") + self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) + self.tokenizer.pad_token_id = self.tokenizer.convert_tokens_to_ids('[PAD]') + if self.tokenizer.chat_template is None: + self.tokenizer.chat_template = SIMPLE_CHAT_TEMPLATE + + + def _setup_dataset(self): + from datasets import load_dataset + + dataset = load_dataset("trl-internal-testing/tldr-preference-sft-trl-style") + print("processing") + train_dataset = dataset["train"] + eval_dataset = dataset["validation"] + + def prepare_dataset(dataset, tokenizer): + def tokenize(element): + input_ids = tokenizer.apply_chat_template( + element["messages"][:1], + padding=False, + add_generation_prompt=True, + ) + return {"input_ids": input_ids, "lengths": len(input_ids), "prompt": element["messages"][:1]} + + return dataset.map( + tokenize, + remove_columns=dataset.column_names, + num_proc=4, + ) + + with PartialState().local_main_process_first(): + train_dataset = prepare_dataset(train_dataset, self.tokenizer) + if eval_dataset is not None: + eval_dataset = prepare_dataset(eval_dataset, self.tokenizer) + train_dataset = train_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) + if eval_dataset is not None: + eval_dataset = eval_dataset.filter(lambda x: x["lengths"] <= 512, num_proc=4) + + assert train_dataset[0]["input_ids"][-1] != self.tokenizer.eos_token_id, "The last token should not be an EOS token" + + self.train_dataset = train_dataset + self.val_dataset = eval_dataset + print(f"Dataset loaded and processed: {len(self.train_dataset)} train, {len(self.val_dataset or [])} validation") + + def _create_quantization_config(self): + self.quant_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.bfloat16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type="nf4" + ) + + def _setup_grpo_trainer(self): + num_processes = self.accelerator.num_processes + global_batch_size = self.args.per_device_train_batch_size * num_processes + + num_generations = 4 # manually chosen value + print(f"Using num_generations = {num_generations} (global_batch_size = {global_batch_size})") + + policy_model = AutoModelForCausalLM.from_pretrained( + "/data/disk0/models/EleutherAI_pythia-1b-deduped__sft__tldr", + torch_dtype='auto', + num_labels=1, + ) + + reward_model = AutoModelForSequenceClassification.from_pretrained( + "/data/disk0/models/EleutherAI_pythia-1b-deduped__reward__tldr", + torch_dtype='auto', + num_labels=1, + ) + + training_args = GRPOConfig( + logging_steps = 1, + report_to = "wandb", + per_device_train_batch_size=self.args.per_device_train_batch_size, + per_device_eval_batch_size=self.args.per_device_eval_batch_size, + gradient_accumulation_steps=self.args.gradient_accumulation_steps, + num_train_epochs=self.args.num_train_epochs, + learning_rate=self.args.learning_rate, + output_dir=self.args.output_dir, + save_steps=self.args.save_steps, + num_generations=num_generations + ) + + self.grpo_trainer = GRPOTrainer( + args=training_args, + model=policy_model, + reward_funcs=reward_model, + processing_class=self.tokenizer, + reward_processing_classes=self.tokenizer, + train_dataset=self.train_dataset, + eval_dataset=self.val_dataset, + ) + print("GRPOtrainer setup complete") + + def train(self): + try: + print("Starting GRPO training...") + train_stats = self.grpo_trainer.train() + return train_stats + except Exception as e: + print(f"Training error: {str(e)}") + raise diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index c7fe603..a0767db 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -15,6 +15,7 @@ from peft import prepare_model_for_kbit_training from trl.trainer.utils import disable_dropout_in_model from accelerate import PartialState, Accelerator +import copy import wandb from sotopia_rl.data import PPODataset @@ -104,6 +105,7 @@ def _setup_generation_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_gen_ref = prepare_model_for_kbit_training(base_gen_ref) self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_ref, self.args.ref_adapter_path, @@ -113,14 +115,15 @@ def _setup_generation_models(self): if self.args.use_lora_train_ppo: - base_gen_policy = AutoModelForCausalLM.from_pretrained( + self.base_gen_policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, torch_dtype='auto', quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + self.base_gen_policy = prepare_model_for_kbit_training(self.base_gen_policy) self.policy = PeftModelForCausalLM.from_pretrained( - base_gen_policy, + self.base_gen_policy, self.args.policy_adapter_path, is_trainable=True, adapter_name="policy_adapter" @@ -173,6 +176,7 @@ def _setup_classification_models(self): quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) + base_value_model = prepare_model_for_kbit_training(base_value_model) self.value_model = PeftModelForSequenceClassification.from_pretrained( base_value_model, self.args.value_adapter_path, @@ -204,7 +208,6 @@ def _setup_classification_models(self): def _setup_ppo_trainer(self): training_args = PPOConfig( - vf_coef=0.5, per_device_train_batch_size=self.args.per_device_train_batch_size, per_device_eval_batch_size=self.args.per_device_eval_batch_size, num_mini_batches=self.args.num_mini_batches, @@ -226,8 +229,8 @@ def _setup_ppo_trainer(self): self.ppo_trainer = PPOTrainer( args=training_args, model=self.policy, + ref_model=self.base_gen_policy, processing_class=self.tokenizer, - ref_model=self.ref_policy, reward_model=self.reward_model, value_model=self.value_model, train_dataset=self.train_dataset, From 69fe736ebe90050e749eecc1a45da98d0f68cb0b Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Wed, 23 Apr 2025 15:24:52 +0800 Subject: [PATCH 18/26] update --- scripts/accelerate_config_grpo.yaml | 2 +- scripts/accelerate_config_ppo.yaml | 2 +- scripts/train_grpo.py | 2 + scripts/train_grpo.sh | 14 ++-- scripts/train_ppo.sh | 16 ++-- scripts/train_sft.sh | 17 ++-- sotopia_rl/grpo_trainer.py | 126 +++++++++++++++++++++------- sotopia_rl/ppo_trainer.py | 15 +--- 8 files changed, 127 insertions(+), 67 deletions(-) diff --git a/scripts/accelerate_config_grpo.yaml b/scripts/accelerate_config_grpo.yaml index ac91a13..b80a54d 100644 --- a/scripts/accelerate_config_grpo.yaml +++ b/scripts/accelerate_config_grpo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 6 +num_processes: 4 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index f025f6d..977e4a5 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 8 +num_processes: 4 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/train_grpo.py b/scripts/train_grpo.py index 55a9181..5ef8913 100644 --- a/scripts/train_grpo.py +++ b/scripts/train_grpo.py @@ -31,6 +31,8 @@ help="Maximum length of generated responses") parser.add_argument("--num_generations", type=int, default=4, help="Number of generations for GRPO") + parser.add_argument("--beta", type=float, default=0.04, + help="KL coefficient for GRPO") # Adapter parameters parser.add_argument("--policy_adapter_path", type=str, default=None, diff --git a/scripts/train_grpo.sh b/scripts/train_grpo.sh index b8abdf2..3cf3be1 100644 --- a/scripts/train_grpo.sh +++ b/scripts/train_grpo.sh @@ -1,4 +1,4 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 accelerate launch \ +CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_grpo.yaml \ --main_process_port 29511 \ /data/disk0/sotopia-rl/scripts/train_grpo.py \ @@ -6,13 +6,13 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 accelerate launch \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ --learning_rate 5e-6 \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --gradient_accumulation_steps 1 \ + --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 8 \ --grpo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_grpo_epochs 2 \ - --num_train_epochs 5 \ --use_lora_train_grpo \ - --num_generations 4 \ - --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default + --num_generations 16 \ + --beta 1e-4 \ + --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default_beta_1e-4 \ No newline at end of file diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 74f7fb9..368fa12 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,14 +1,14 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ +CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29529 \ + --main_process_port 29509 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-350 \ - --value_adapter_path /data/disk0/sotopia-rl/ppo_token_length_normalized_checkpoint_510_value_adapter \ - --learning_rate 5e-5 \ - --per_device_train_batch_size 6 \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/ppo_token_length_normalized/checkpoint-630 \ + --learning_rate 1e-5 \ + --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 1 \ --num_mini_batches 1 \ diff --git a/scripts/train_sft.sh b/scripts/train_sft.sh index 4836d6a..4ab5944 100644 --- a/scripts/train_sft.sh +++ b/scripts/train_sft.sh @@ -1,8 +1,8 @@ -CUDA_VISIBLE_DEVICES=7,8 accelerate launch \ - --config_file /data/haofeiy2/sotopia-rl/scripts/accelerate_config_sft.yaml \ +CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_sft.yaml \ --main_process_port 29512 \ - /data/haofeiy2/sotopia-rl/scripts/train_sft.py \ - --model_name /mnt/data_from_server1/models/Qwen2.5-7B-Instruct \ + /data/disk0/sotopia-rl/scripts/train_sft.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --learning_rate 1e-4 \ --max_length 4096 \ --train_batch_size 1 \ @@ -10,9 +10,8 @@ CUDA_VISIBLE_DEVICES=7,8 accelerate launch \ --accumulation_steps 6 \ --num_epochs 20 \ --use_lora \ - --evaluation_steps 100 \ - --sft_data_path /data/haofeiy2/sotopia-rl/data/sotopia_pi_round1_qwen_sft_pi_with_instruct_string.json \ - --template_path /data/haofeiy2/sotopia-rl/evals/qwen2.5-7b.jinja \ - --lora_checkpoint_path /data/haofeiy2/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --checkpoint_dir /data/haofeiy2/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi \ + --evaluation_steps 50 \ + --sft_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_pi_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --checkpoint_dir /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi \ --use_qlora \ No newline at end of file diff --git a/sotopia_rl/grpo_trainer.py b/sotopia_rl/grpo_trainer.py index beafa71..7bb2025 100644 --- a/sotopia_rl/grpo_trainer.py +++ b/sotopia_rl/grpo_trainer.py @@ -20,8 +20,8 @@ from typing import List os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" -os.environ['NCCL_P2P_DISABLE'] = '1' -os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'DETAIL' +os.environ["NCCL_P2P_DISABLE"] = "1" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "DETAIL" class SotopiaGRPOTrainer: @@ -29,7 +29,8 @@ def __init__(self, args, accelerator: Accelerator): self.args = args self.accelerator = accelerator - self._init_wandb() + if accelerator.is_main_process: + self._init_wandb() self._setup_tokenizer() self._setup_dataset() self._create_quantization_config() @@ -43,51 +44,64 @@ def save_model(self, output_dir: str, _internal_call: bool = False): self.tokenizer.save_pretrained(output_dir) print(f"Saved PEFT model to {output_dir}") - self.grpo_trainer.save_model = save_model.__get__(self.grpo_trainer, type(self.grpo_trainer)) + self.grpo_trainer.save_model = save_model.__get__( + self.grpo_trainer, type(self.grpo_trainer) + ) def _init_wandb(self): wandb.init( project=self.args.wandb_project, name=self.args.wandb_run_name, - config={k: v for k, v in vars(self.args).items() if isinstance(v, (int, float, str))} + config={ + k: v + for k, v in vars(self.args).items() + if isinstance(v, (int, float, str)) + }, ) def _setup_tokenizer(self): - self.tokenizer = AutoTokenizer.from_pretrained(self.args.model_name, padding_side="left") - self.tokenizer.add_special_tokens({'pad_token': '[PAD]'}) - self.tokenizer.pad_token_id = self.tokenizer.convert_tokens_to_ids('[PAD]') - + self.tokenizer = AutoTokenizer.from_pretrained( + self.args.model_name, padding_side="left" + ) + self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + self.tokenizer.pad_token_id = self.tokenizer.convert_tokens_to_ids("[PAD]") def _setup_dataset(self): - env = Environment(loader=FileSystemLoader("/".join(self.args.template_path.split("/")[:-1]))) + env = Environment( + loader=FileSystemLoader("/".join(self.args.template_path.split("/")[:-1])) + ) template = env.get_template(self.args.template_path.split("/")[-1]) dataset = GRPODataset( data_path=self.args.grpo_data_path, tokenizer=self.tokenizer, template=template, - max_length=self.args.max_length + max_length=self.args.max_length, ) generator = torch.Generator().manual_seed(42) - val_ratio = getattr(self.args, 'val_ratio', 0.05) + val_ratio = getattr(self.args, "val_ratio", 0.05) train_size = min(int(len(dataset) * (1 - val_ratio)), len(dataset) - 2) val_size = len(dataset) - train_size - self.train_dataset, self.val_dataset = random_split(dataset, [train_size, val_size], generator=generator) - print(f"Dataset split: {len(self.train_dataset)} train, {len(self.val_dataset)} validation") + self.train_dataset, self.val_dataset = random_split( + dataset, [train_size, val_size], generator=generator + ) + print( + f"Dataset split: {len(self.train_dataset)} train, {len(self.val_dataset)} validation" + ) def _create_quantization_config(self): self.quant_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4" + bnb_4bit_quant_type="nf4", ) def _setup_policy_models(self): if self.args.use_lora_train_grpo: base_gen_policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, - torch_dtype='auto', + torch_dtype="auto", quantization_config=self.quant_config, device_map=get_kbit_device_map(), ) @@ -95,20 +109,29 @@ def _setup_policy_models(self): base_gen_policy, self.args.policy_adapter_path, is_trainable=True, - adapter_name="policy_adapter" + adapter_name="policy_adapter", ) else: self.policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, - torch_dtype='auto', + torch_dtype="auto", ) self.policy.config.pad_token_id = self.tokenizer.pad_token_id + requires_grad_num = 0 + for name, param in self.policy.named_parameters(): + print(name, param.requires_grad) + if param.requires_grad: + requires_grad_num += 1 + print(f"Number of trainable parameters in policy: {requires_grad_num}") + def _setup_classification_models(self): + if self.args.use_lora_train_grpo: + print("using lora for reward model") base_reward_model = AutoModelForSequenceClassification.from_pretrained( self.args.model_name, - torch_dtype='auto', + torch_dtype="auto", num_labels=1, quantization_config=self.quant_config, device_map=get_kbit_device_map(), @@ -116,27 +139,64 @@ def _setup_classification_models(self): self.reward_model = PeftModelForSequenceClassification.from_pretrained( base_reward_model, self.args.reward_adapter_path, - is_trainable=True, - adapter_name="value_adapter" + is_trainable=False, + adapter_name="value_adapter", ) else: self.reward_model = AutoModelForSequenceClassification.from_pretrained( self.args.model_name, - torch_dtype='auto', + torch_dtype="auto", num_labels=1, ) self.reward_model.config.pad_token_id = self.tokenizer.pad_token_id + def wrapped_reward( + prompts: list[str], completions: list[str], **kwargs + ) -> list[float]: + eos = self.tokenizer.eos_token + completions = [c + eos for c in completions] + + texts = [p + c for p, c in zip(prompts, completions)] + + inputs = self.tokenizer( + text=texts, + return_tensors="pt", + padding=True, + truncation=True, + max_length=4096, + ).to(self.accelerator.device) + + with torch.inference_mode(): + logits = self.reward_model(**inputs).logits[:, 0] + return logits.cpu().tolist() + + self.wrapped_reward = wrapped_reward + for p in self.reward_model.parameters(): + p.requires_grad = False + requires_grad_num = 0 + for name, param in self.reward_model.named_parameters(): + print(name, param.requires_grad) + if param.requires_grad: + requires_grad_num += 1 + print(f"Number of trainable parameters in reward: {requires_grad_num}") + def _setup_grpo_trainer(self): num_processes = self.accelerator.num_processes - global_batch_size = self.args.per_device_train_batch_size * num_processes + global_batch_size = ( + self.args.per_device_train_batch_size + * num_processes + * self.args.gradient_accumulation_steps + ) - num_generations = self.args.num_generations - print(f"Using num_generations = {num_generations} (global_batch_size = {global_batch_size})") + print( + f"Using num_generations = {self.args.num_generations} (global_batch_size = {global_batch_size})" + ) training_args = GRPOConfig( - logging_steps = 1, - report_to = "wandb", + disable_dropout=True, + max_prompt_length=4096, + logging_steps=1, + report_to="wandb", per_device_train_batch_size=self.args.per_device_train_batch_size, per_device_eval_batch_size=self.args.per_device_eval_batch_size, gradient_accumulation_steps=self.args.gradient_accumulation_steps, @@ -144,13 +204,16 @@ def _setup_grpo_trainer(self): learning_rate=self.args.learning_rate, output_dir=self.args.output_dir, save_steps=self.args.save_steps, - num_generations=num_generations + num_generations=self.args.num_generations, + log_completions=True, + wandb_log_unique_prompts=True, + beta=1e-4, ) self.grpo_trainer = GRPOTrainer( args=training_args, model=self.policy, - reward_funcs=self.reward_model, + reward_funcs=self.wrapped_reward, processing_class=self.tokenizer, reward_processing_classes=self.tokenizer, train_dataset=self.train_dataset, @@ -162,7 +225,10 @@ def train(self): try: print("Starting GRPO training...") train_stats = self.grpo_trainer.train() + if self.accelerator.is_main_process: + print("Saving final model checkpoint...") + self.grpo_trainer.save_model(self.args.output_dir) return train_stats except Exception as e: print(f"Training error: {str(e)}") - raise + raise \ No newline at end of file diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index a0767db..865f6d1 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -102,10 +102,7 @@ def _setup_generation_models(self): base_gen_ref = AutoModelForCausalLM.from_pretrained( self.args.model_name, torch_dtype='auto', - quantization_config=self.quant_config, - device_map=get_kbit_device_map(), ) - base_gen_ref = prepare_model_for_kbit_training(base_gen_ref) self.ref_policy = PeftModelForCausalLM.from_pretrained( base_gen_ref, self.args.ref_adapter_path, @@ -118,10 +115,7 @@ def _setup_generation_models(self): self.base_gen_policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, torch_dtype='auto', - quantization_config=self.quant_config, - device_map=get_kbit_device_map(), ) - self.base_gen_policy = prepare_model_for_kbit_training(self.base_gen_policy) self.policy = PeftModelForCausalLM.from_pretrained( self.base_gen_policy, self.args.policy_adapter_path, @@ -173,10 +167,7 @@ def _setup_classification_models(self): self.args.model_name, torch_dtype='auto', num_labels=1, - quantization_config=self.quant_config, - device_map=get_kbit_device_map(), ) - base_value_model = prepare_model_for_kbit_training(base_value_model) self.value_model = PeftModelForSequenceClassification.from_pretrained( base_value_model, self.args.value_adapter_path, @@ -208,6 +199,7 @@ def _setup_classification_models(self): def _setup_ppo_trainer(self): training_args = PPOConfig( + max_grad_norm=0.5, per_device_train_batch_size=self.args.per_device_train_batch_size, per_device_eval_batch_size=self.args.per_device_eval_batch_size, num_mini_batches=self.args.num_mini_batches, @@ -224,12 +216,13 @@ def _setup_ppo_trainer(self): ddp_find_unused_parameters=True, response_length=self.args.response_length, stop_token='eos', + kl_estimator='k3', ) self.ppo_trainer = PPOTrainer( args=training_args, model=self.policy, - ref_model=self.base_gen_policy, + ref_model=self.ref_policy, processing_class=self.tokenizer, reward_model=self.reward_model, value_model=self.value_model, @@ -245,4 +238,4 @@ def train(self): return train_stats except Exception as e: print(f"Training error: {str(e)}") - raise + raise \ No newline at end of file From 779e2a1b97654700fa6cd41a27b9318ec976f1c8 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Thu, 24 Apr 2025 02:12:50 +0800 Subject: [PATCH 19/26] update --- scripts/accelerate_config_ppo.yaml | 2 +- scripts/train_ppo.sh | 8 ++++---- scripts/train_sft.sh | 5 ++--- sotopia_rl/ppo_trainer.py | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index 977e4a5..afe8d0c 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 4 +num_processes: 1 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 368fa12..b3189e3 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,13 +1,13 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ +CUDA_VISIBLE_DEVICES=3 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29509 \ + --main_process_port 29539 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/ppo_token_length_normalized/checkpoint-630 \ - --learning_rate 1e-5 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 1 \ diff --git a/scripts/train_sft.sh b/scripts/train_sft.sh index 4ab5944..50018fb 100644 --- a/scripts/train_sft.sh +++ b/scripts/train_sft.sh @@ -11,7 +11,6 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ --num_epochs 20 \ --use_lora \ --evaluation_steps 50 \ - --sft_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_pi_with_instruct_string.json \ + --sft_data_path /data/disk0/sotopia-rl/data/sotopia_pi_bc_episodes_sft.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ - --checkpoint_dir /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi \ - --use_qlora \ No newline at end of file + --checkpoint_dir /data/disk0/sotopia-rl/sft_qwen25_7b_bc \ No newline at end of file diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 865f6d1..bed1946 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -199,7 +199,6 @@ def _setup_classification_models(self): def _setup_ppo_trainer(self): training_args = PPOConfig( - max_grad_norm=0.5, per_device_train_batch_size=self.args.per_device_train_batch_size, per_device_eval_batch_size=self.args.per_device_eval_batch_size, num_mini_batches=self.args.num_mini_batches, @@ -217,6 +216,7 @@ def _setup_ppo_trainer(self): response_length=self.args.response_length, stop_token='eos', kl_estimator='k3', + kl_coef=1e-5, ) self.ppo_trainer = PPOTrainer( From 04e17b1cb8773fdab4dda3713eba76d1d90ae94b Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Thu, 24 Apr 2025 02:14:09 +0800 Subject: [PATCH 20/26] upload a working script --- scripts/train_ppo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index b3189e3..b495f9f 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -10,7 +10,7 @@ CUDA_VISIBLE_DEVICES=3 accelerate launch \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ - --gradient_accumulation_steps 1 \ + --gradient_accumulation_steps 32 \ --num_mini_batches 1 \ --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ From 766f7ac5877e313be08eca2f6b07337fedc967b0 Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Fri, 25 Apr 2025 06:05:58 +0800 Subject: [PATCH 21/26] I think I finish the code, the bug is because overtraining SFT, I use new SFT ckpt for PPO --- scripts/accelerate_config_ppo.yaml | 2 +- scripts/accelerate_config_sft.yaml | 2 +- scripts/train_ppo.sh | 177 ++++++++++++++++++++++++++++- scripts/train_sft.sh | 29 ++++- sotopia_rl/data.py | 2 +- sotopia_rl/ppo_trainer.py | 1 - 6 files changed, 199 insertions(+), 14 deletions(-) diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index afe8d0c..f025f6d 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 1 +num_processes: 8 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/accelerate_config_sft.yaml b/scripts/accelerate_config_sft.yaml index e517fed..5b42349 100644 --- a/scripts/accelerate_config_sft.yaml +++ b/scripts/accelerate_config_sft.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 2 +num_processes: 8 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index b495f9f..1ca04e9 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,10 +1,177 @@ -CUDA_VISIBLE_DEVICES=3 accelerate launch \ +# parameter I used for final PPO checkpoint +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ - --main_process_port 29539 \ + --main_process_port 29439 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_pi_round1_qwen_sft_pi/checkpoint-50 \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl + + + +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29439 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step104_default_kl + +CUDA_VISIBLE_DEVICES=4,5 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29449 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step30_default_kl + + +CUDA_VISIBLE_DEVICES=2,3 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29469 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step50_default_kl + + +CUDA_VISIBLE_DEVICES=0,1 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29499 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step70_default_kl + + +CUDA_VISIBLE_DEVICES=5 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29549 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 32 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/ + +CUDA_VISIBLE_DEVICES=6 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29559 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --learning_rate 1e-6 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 32 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 5 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt500 + +CUDA_VISIBLE_DEVICES=7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29569 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ @@ -19,4 +186,4 @@ CUDA_VISIBLE_DEVICES=3 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized + --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt700 diff --git a/scripts/train_sft.sh b/scripts/train_sft.sh index 50018fb..9e767d9 100644 --- a/scripts/train_sft.sh +++ b/scripts/train_sft.sh @@ -1,6 +1,25 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ +# parameter I used for final SFT checkpoint, I use ckpt 160 +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_sft.yaml \ - --main_process_port 29512 \ + --main_process_port 29112 \ + /data/disk0/sotopia-rl/scripts/train_sft.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --learning_rate 1e-5 \ + --max_length 4096 \ + --train_batch_size 1 \ + --val_batch_size 4 \ + --accumulation_steps 1 \ + --num_epochs 1 \ + --use_lora \ + --evaluation_steps 5 \ + --sft_data_path /data/disk0/sotopia-rl/data/sft_round_1_bc_data_top_2.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --checkpoint_dir /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch + + +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_sft.yaml \ + --main_process_port 29112 \ /data/disk0/sotopia-rl/scripts/train_sft.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --learning_rate 1e-4 \ @@ -10,7 +29,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ --accumulation_steps 6 \ --num_epochs 20 \ --use_lora \ - --evaluation_steps 50 \ - --sft_data_path /data/disk0/sotopia-rl/data/sotopia_pi_bc_episodes_sft.json \ + --evaluation_steps 10 \ + --sft_data_path /data/disk0/sotopia-rl/data/sft_round_1_bc_data_top_2.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ - --checkpoint_dir /data/disk0/sotopia-rl/sft_qwen25_7b_bc \ No newline at end of file + --checkpoint_dir /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt \ No newline at end of file diff --git a/sotopia_rl/data.py b/sotopia_rl/data.py index b7fff53..e440149 100644 --- a/sotopia_rl/data.py +++ b/sotopia_rl/data.py @@ -198,4 +198,4 @@ def __getitem__(self, idx: int) -> Dict[str, Any]: return { "prompt": rendered_prompt, "completion": item["output"] - } + } \ No newline at end of file diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index bed1946..16e56a6 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -216,7 +216,6 @@ def _setup_ppo_trainer(self): response_length=self.args.response_length, stop_token='eos', kl_estimator='k3', - kl_coef=1e-5, ) self.ppo_trainer = PPOTrainer( From 52aade3d8aad12bed791671c7076620087673edf Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sat, 26 Apr 2025 12:15:37 +0800 Subject: [PATCH 22/26] update --- scripts/accelerate_config_grpo.yaml | 2 +- scripts/train_grpo.sh | 2 +- scripts/train_ppo.py | 2 +- scripts/train_ppo.sh | 54 +++++++++++++++++++++-------- sotopia_rl/ppo_trainer.py | 9 ++--- 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/scripts/accelerate_config_grpo.yaml b/scripts/accelerate_config_grpo.yaml index b80a54d..0704946 100644 --- a/scripts/accelerate_config_grpo.yaml +++ b/scripts/accelerate_config_grpo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 4 +num_processes: 1 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/train_grpo.sh b/scripts/train_grpo.sh index 3cf3be1..889680e 100644 --- a/scripts/train_grpo.sh +++ b/scripts/train_grpo.sh @@ -1,4 +1,4 @@ -CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \ +CUDA_VISIBLE_DEVICES=7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_grpo.yaml \ --main_process_port 29511 \ /data/disk0/sotopia-rl/scripts/train_grpo.py \ diff --git a/scripts/train_ppo.py b/scripts/train_ppo.py index a8bdfec..b769cbe 100644 --- a/scripts/train_ppo.py +++ b/scripts/train_ppo.py @@ -32,7 +32,7 @@ help="Number of steps to accumulate gradients before performing an update") parser.add_argument("--val_ratio", type=float, default=0.05, help="Ratio of validation data") - parser.add_argument("--response_length", type=int, default=128, + parser.add_argument("--response_length", type=int, default=256, help="Maximum length of generated responses") parser.add_argument("--local_rollout_forward_batch_size", type=int, default=16, help="Batch size for local rollout forward pass") diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 1ca04e9..6071cee 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -4,23 +4,47 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --main_process_port 29439 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ - --learning_rate 1e-6 \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ + --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_pretrain_value_model/checkpoint-100 \ + --learning_rate 5e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ - --num_train_epochs 5 \ + --num_train_epochs 30 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_with_1e-4_vf_coef_0425 + +# param for pretrain value adapter +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ + --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --main_process_port 29439 \ + /data/disk0/sotopia-rl/scripts/train_ppo.py \ + --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ + --value_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ + --learning_rate 1e-4 \ + --per_device_train_batch_size 1 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 4 \ + --num_mini_batches 1 \ + --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --num_train_epochs 30 \ + --max_length 4096 \ + --num_ppo_epochs 2 \ + --gamma 1.00 \ + --use_lora_train_ppo \ + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_pretrain_value_model @@ -29,14 +53,14 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --main_process_port 29439 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_only_1_epoch/checkpoint-160 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --learning_rate 1e-6 \ + --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_checkpoint-800 \ + --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_pretrained_value_model \ + --learning_rate 1e-5 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ - --gradient_accumulation_steps 4 \ + --gradient_accumulation_steps 2 \ --num_mini_batches 1 \ --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ @@ -45,9 +69,9 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step104_default_kl + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_with_pretrained_value_model -CUDA_VISIBLE_DEVICES=4,5 accelerate launch \ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29449 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 16e56a6..10df0da 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -99,16 +99,10 @@ def _create_quantization_config(self): ) def _setup_generation_models(self): - base_gen_ref = AutoModelForCausalLM.from_pretrained( + self.ref_policy = AutoModelForCausalLM.from_pretrained( self.args.model_name, torch_dtype='auto', ) - self.ref_policy = PeftModelForCausalLM.from_pretrained( - base_gen_ref, - self.args.ref_adapter_path, - is_trainable=False, - adapter_name="ref_adapter" - ) if self.args.use_lora_train_ppo: @@ -216,6 +210,7 @@ def _setup_ppo_trainer(self): response_length=self.args.response_length, stop_token='eos', kl_estimator='k3', + vf_coef=1e-4, ) self.ppo_trainer = PPOTrainer( From ac279ccd93b6be0d0b61be66e5e9f8cd9b56547c Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Sun, 27 Apr 2025 09:53:44 +0800 Subject: [PATCH 23/26] I think this setting has normal reward increase now --- scripts/train_ppo.sh | 6 +++--- sotopia_rl/ppo_trainer.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 6071cee..0f22f7e 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -8,7 +8,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_pretrain_value_model/checkpoint-100 \ - --learning_rate 5e-6 \ + --learning_rate 4e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ @@ -20,7 +20,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_with_1e-4_vf_coef_0425 + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model # param for pretrain value adapter CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ @@ -44,7 +44,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_pretrain_value_model + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 10df0da..955bfc6 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -211,6 +211,8 @@ def _setup_ppo_trainer(self): stop_token='eos', kl_estimator='k3', vf_coef=1e-4, + kl_coef=1e-2, + max_grad_norm=0.5, ) self.ppo_trainer = PPOTrainer( From 5104561d5f46bc2b5d9c384e458a6b5bec86c02a Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Mon, 28 Apr 2025 05:02:46 +0800 Subject: [PATCH 24/26] support rm direct and successfully make the reward improving --- scripts/accelerate_config_grpo.yaml | 2 +- scripts/train_grpo.sh | 8 ++++---- scripts/train_ppo.sh | 14 +++++++------- sotopia_rl/ppo_trainer.py | 21 ++++++++++++--------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/scripts/accelerate_config_grpo.yaml b/scripts/accelerate_config_grpo.yaml index 0704946..b80a54d 100644 --- a/scripts/accelerate_config_grpo.yaml +++ b/scripts/accelerate_config_grpo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 1 +num_processes: 4 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/train_grpo.sh b/scripts/train_grpo.sh index 889680e..48ebf77 100644 --- a/scripts/train_grpo.sh +++ b/scripts/train_grpo.sh @@ -1,10 +1,10 @@ -CUDA_VISIBLE_DEVICES=7 accelerate launch \ +CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_grpo.yaml \ --main_process_port 29511 \ /data/disk0/sotopia-rl/scripts/train_grpo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_relationship_social_rules_conversation_behavior_4_26/checkpoint-7200 \ --learning_rate 5e-6 \ --per_device_train_batch_size 4 \ --per_device_eval_batch_size 4 \ @@ -14,5 +14,5 @@ CUDA_VISIBLE_DEVICES=7 accelerate launch \ --num_grpo_epochs 2 \ --use_lora_train_grpo \ --num_generations 16 \ - --beta 1e-4 \ - --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default_beta_1e-4 \ No newline at end of file + --beta 0.04 \ + --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_goal_w_relationship_social_rules_conversation_behavior_default_beta_004 \ No newline at end of file diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 0f22f7e..8e2e4a7 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -6,9 +6,9 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ - --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_with_pretrained_value_model_with_conv_constrain_rm_pretrain_value_model/checkpoint-100 \ - --learning_rate 4e-6 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ + --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm/checkpoint-90 \ + --learning_rate 3e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ @@ -20,10 +20,10 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model_gamma_099_direct_rm # param for pretrain value adapter -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ +CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29439 \ /data/disk0/sotopia-rl/scripts/train_ppo.py \ @@ -32,7 +32,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ --value_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ - --learning_rate 1e-4 \ + --learning_rate 5e-5 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ @@ -44,7 +44,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model + --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 955bfc6..3df422e 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -35,7 +35,7 @@ def __init__(self, args): self._setup_classification_models() self._setup_ppo_trainer() - for m in [self.policy, self.ref_policy]: + for m in [self.policy]: m.config.use_cache = False for m in [self.value_model, self.reward_model]: m.config.use_cache = False @@ -131,11 +131,15 @@ def _setup_generation_models(self): requires_grad_num += 1 print(f"Number of trainable parameters in policy: {requires_grad_num}") - requires_grad_num = 0 - for name, param in self.ref_policy.named_parameters(): - if param.requires_grad: - requires_grad_num += 1 - print(f"Number of trainable parameters in ref policy: {requires_grad_num}") + + #for name, param in self.policy.named_parameters(): + # if self.policy.active_adapter in name: + # param.requires_grad = False + #requires_grad_num = 0 + #for name, param in self.ref_policy.named_parameters(): + # if param.requires_grad: + # requires_grad_num += 1 + #print(f"Number of trainable parameters in ref policy: {requires_grad_num}") def _setup_classification_models(self): base_reward_model = AutoModelForSequenceClassification.from_pretrained( @@ -210,9 +214,8 @@ def _setup_ppo_trainer(self): response_length=self.args.response_length, stop_token='eos', kl_estimator='k3', - vf_coef=1e-4, - kl_coef=1e-2, - max_grad_norm=0.5, + vf_coef=1e-3, + kl_coef=0.05, ) self.ppo_trainer = PPOTrainer( From 1da0de99d26d412932bcd7c7c23399ec68e07c5e Mon Sep 17 00:00:00 2001 From: robotMonkeyButler Date: Tue, 29 Apr 2025 08:57:19 +0800 Subject: [PATCH 25/26] update --- scripts/accelerate_config_grpo.yaml | 2 +- scripts/train_grpo.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/accelerate_config_grpo.yaml b/scripts/accelerate_config_grpo.yaml index b80a54d..5b42349 100644 --- a/scripts/accelerate_config_grpo.yaml +++ b/scripts/accelerate_config_grpo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 4 +num_processes: 8 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/train_grpo.sh b/scripts/train_grpo.sh index 48ebf77..0eaf72c 100644 --- a/scripts/train_grpo.sh +++ b/scripts/train_grpo.sh @@ -1,10 +1,10 @@ -CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_grpo.yaml \ --main_process_port 29511 \ /data/disk0/sotopia-rl/scripts/train_grpo.py \ --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_relationship_social_rules_conversation_behavior_4_26/checkpoint-7200 \ + --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ --learning_rate 5e-6 \ --per_device_train_batch_size 4 \ --per_device_eval_batch_size 4 \ @@ -15,4 +15,4 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 accelerate launch \ --use_lora_train_grpo \ --num_generations 16 \ --beta 0.04 \ - --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_goal_w_relationship_social_rules_conversation_behavior_default_beta_004 \ No newline at end of file + --output_dir /data/disk0/sotopia-rl/grpo_rm_reward_direct_default_beta_004 \ No newline at end of file From 8eb1138e68378f1078e73827a702cb2f734be5fd Mon Sep 17 00:00:00 2001 From: Haofei Yu Date: Mon, 28 Apr 2025 20:24:34 -0500 Subject: [PATCH 26/26] use ncsa gpu --- scripts/accelerate_config_grpo.yaml | 2 +- scripts/accelerate_config_ppo.yaml | 2 +- scripts/train_ppo.sh | 184 ++++++++++++++-------------- sotopia_rl/ppo_trainer.py | 2 +- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/scripts/accelerate_config_grpo.yaml b/scripts/accelerate_config_grpo.yaml index b80a54d..ac91a13 100644 --- a/scripts/accelerate_config_grpo.yaml +++ b/scripts/accelerate_config_grpo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 4 +num_processes: 6 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/accelerate_config_ppo.yaml b/scripts/accelerate_config_ppo.yaml index f025f6d..775922f 100644 --- a/scripts/accelerate_config_ppo.yaml +++ b/scripts/accelerate_config_ppo.yaml @@ -8,7 +8,7 @@ machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 -num_processes: 8 +num_processes: 6 rdzv_backend: static # Keep this unless running multi-node same_network: true tpu_env: [] diff --git a/scripts/train_ppo.sh b/scripts/train_ppo.sh index 8e2e4a7..6c04e89 100644 --- a/scripts/train_ppo.sh +++ b/scripts/train_ppo.sh @@ -1,213 +1,213 @@ # parameter I used for final PPO checkpoint CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29439 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ - --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm/checkpoint-90 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_reward_direct_default_without_that_n_error_as_the_end/checkpoint-4480 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm/checkpoint-90 \ --learning_rate 3e-6 \ - --per_device_train_batch_size 1 \ + --per_device_train_batch_size 3 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 30 \ --max_length 4096 \ --num_ppo_epochs 2 \ - --gamma 1.00 \ + --gamma 0.99 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model_gamma_099_direct_rm + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_with_pretrained_value_model_gamma_099_direct_rm # param for pretrain value adapter CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29439 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_goal_w_conversation_behavior_4_23/checkpoint-9400 \ --learning_rate 5e-5 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 30 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_step1500_for_pretrained_value_model_direct_rm CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29439 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_checkpoint-800 \ - --value_adapter_path /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_pretrained_value_model \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_sft_round_1_bc_data_top_2/checkpoint-1500 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_checkpoint-800 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_pretrained_value_model \ --learning_rate 1e-5 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 2 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_with_pretrained_value_model + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_top_2_sft_1_epoch_step160_default_kl_token_length_with_pretrained_value_model CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29449 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step30_default_kl + --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step30_default_kl CUDA_VISIBLE_DEVICES=2,3 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29469 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-50 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step50_default_kl + --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step50_default_kl CUDA_VISIBLE_DEVICES=0,1 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29499 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-70 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 4 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/grpo_top_2_sft_step70_default_kl + --output_dir /projects/bdpw/haofeiy/sotopia-rl/grpo_top_2_sft_step70_default_kl CUDA_VISIBLE_DEVICES=5 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29549 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_round_1_bc_data_top_2_ckpt/checkpoint-30 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 32 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ CUDA_VISIBLE_DEVICES=6 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29559 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-500 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 32 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt500 + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt500 CUDA_VISIBLE_DEVICES=7 accelerate launch \ - --config_file /data/disk0/sotopia-rl/scripts/accelerate_config_ppo.yaml \ + --config_file /projects/bdpw/haofeiy/sotopia-rl/scripts/accelerate_config_ppo.yaml \ --main_process_port 29569 \ - /data/disk0/sotopia-rl/scripts/train_ppo.py \ - --model_name /data/disk0/models/Qwen2.5-7B-Instruct \ - --policy_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ - --ref_adapter_path /data/disk0/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ - --reward_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ - --value_adapter_path /data/disk0/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + /projects/bdpw/haofeiy/sotopia-rl/scripts/train_ppo.py \ + --model_name /projects/bdpw/haofeiy/models/Qwen2.5-7B-Instruct \ + --policy_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ + --ref_adapter_path /projects/bdpw/haofeiy/sotopia-rl/sft_qwen25_7b_bc/checkpoint-700 \ + --reward_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ + --value_adapter_path /projects/bdpw/haofeiy/sotopia-rl/rm_token_length_normalized/checkpoint-500 \ --learning_rate 1e-6 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ --gradient_accumulation_steps 32 \ --num_mini_batches 1 \ - --ppo_data_path /data/disk0/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ - --template_path /data/disk0/sotopia-rl/evals/qwen2.5-7b.jinja \ + --ppo_data_path /projects/bdpw/haofeiy/sotopia-rl/data/sotopia_pi_round1_qwen_sft_all_with_instruct_string.json \ + --template_path /projects/bdpw/haofeiy/sotopia-rl/evals/qwen2.5-7b.jinja \ --num_train_epochs 5 \ --max_length 4096 \ --num_ppo_epochs 2 \ --gamma 1.00 \ --use_lora_train_ppo \ - --output_dir /data/disk0/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt700 + --output_dir /projects/bdpw/haofeiy/sotopia-rl/ppo_token_length_normalized_with_sft_testing_ckpt700 diff --git a/sotopia_rl/ppo_trainer.py b/sotopia_rl/ppo_trainer.py index 3df422e..dd4a896 100644 --- a/sotopia_rl/ppo_trainer.py +++ b/sotopia_rl/ppo_trainer.py @@ -221,7 +221,7 @@ def _setup_ppo_trainer(self): self.ppo_trainer = PPOTrainer( args=training_args, model=self.policy, - ref_model=self.ref_policy, + ref_model=copy.deepcopy(self.policy), processing_class=self.tokenizer, reward_model=self.reward_model, value_model=self.value_model,