diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index 9c0d68f..483a8dc 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -1,19 +1,19 @@ # SDPO config for sparse parity challenge on 1x A100 80GB (a100-backup-1) -# Adapted from autoresearch_sdpo.yaml for single GPU + shorter context +# Qwen3-8B + LoRA r32, LR 5e-5, 4x parallel tool calls, 0.45 GPU mem util # Usage: bash scripts/run_sparse_parity.sh [experiment-name] defaults: - sdpo - _self_ -max_model_len: 32768 +max_model_len: 16384 data: train_files: ["${vars.dir}/data/sparse_parity/train.parquet"] val_files: ["${vars.dir}/data/sparse_parity/test.parquet"] train_batch_size: 8 - max_prompt_length: 8192 - max_response_length: 24576 + max_prompt_length: 4096 + max_response_length: 12288 apply_chat_template_kwargs: {"enable_thinking": true} return_raw_chat: True filter_overlong_prompts: True @@ -22,17 +22,18 @@ data: actor_rollout_ref: model: - path: Qwen/Qwen3-14B + path: Qwen/Qwen3-8B trust_remote_code: True lora_rank: 32 lora_alpha: 16 target_modules: all-linear + enable_gradient_checkpointing: true override_config: attn_implementation: flash_attention_2 actor: ppo_micro_batch_size_per_gpu: 1 ppo_mini_batch_size: 8 - ppo_max_token_len_per_gpu: ${max_model_len} + ppo_max_token_len_per_gpu: 16384 use_dynamic_bsz: False ulysses_sequence_parallel_size: 1 strategy: fsdp2 @@ -45,7 +46,7 @@ actor_rollout_ref: loss_mode: sdpo self_distillation: teacher_update_rate: 0.01 - max_reprompt_len: 8192 + max_reprompt_len: 4096 is_clip: 2.0 alpha: 1.0 distillation_topk: 20 @@ -66,29 +67,22 @@ actor_rollout_ref: optim: optimizer: AdamW optimizer_impl: torch.optim - lr: 1e-6 + lr: 3e-5 lr_warmup_steps: 5 rollout: name: vllm n: 1 temperature: 1.0 tensor_model_parallel_size: 1 - gpu_memory_utilization: 0.55 + gpu_memory_utilization: 0.35 max_model_len: ${max_model_len} max_num_batched_tokens: ${max_model_len} calculate_log_probs: True - engine_kwargs: - vllm: - hf_overrides: - rope_scaling: - rope_type: yarn - factor: 1.0 - original_max_position_embeddings: 32768 multi_turn: enable: True max_user_turns: 10 max_assistant_turns: 10 - max_parallel_calls: 1 + max_parallel_calls: 4 max_tool_response_length: 10000 tool_response_truncate_side: right tool_config_path: ${vars.dir}/configs/sparse_parity_bash_tool.yaml @@ -119,7 +113,7 @@ critic: strategy: fsdp2 ulysses_sequence_parallel_size: 1 model: - path: Qwen/Qwen3-14B + path: Qwen/Qwen3-8B custom_reward_function: path: ${vars.dir}/training/reward.py @@ -130,7 +124,7 @@ reward_model: trainer: project_name: sparse-parity-sdpo group_name: sparse-parity-sdpo - experiment_name: qwen3-14b-sdpo-lora32 + experiment_name: qwen3-8b-sdpo-lora32 n_gpus_per_node: 1 nnodes: 1 total_epochs: 1000 diff --git a/training/run_sdpo.py b/training/run_sdpo.py index 97bb76d..70cf0f1 100644 --- a/training/run_sdpo.py +++ b/training/run_sdpo.py @@ -102,6 +102,11 @@ def run(self, config): def main(config): from verl.utils.device import auto_set_device auto_set_device(config) + # Force ppo_max_token_len_per_gpu to match max_model_len — Hydra structured + # config silently ignores YAML/CLI overrides for this interpolated field. + from omegaconf import OmegaConf, flag_override + with flag_override(config, "struct", False): + config.actor_rollout_ref.actor.ppo_max_token_len_per_gpu = config.max_model_len task_runner_class = ray.remote(num_cpus=1)(PatchedTaskRunner) run_ppo(config, task_runner_class=task_runner_class)