From 4221d9f1c3b43d025ca61653ca66a3ac3290d18a Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 10 Apr 2026 09:21:13 -0700 Subject: [PATCH 01/10] Switch sparse parity SDPO to Qwen3-8B with speed improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Model: Qwen3-14B → Qwen3-8B (~8B params, ~16GB bf16, fits in 80GB) - LR: 1e-6 → 5e-5 (50x increase, appropriate for LoRA) - max_parallel_calls: 1 → 4 (parallelize tool calls in rollout) - gpu_memory_utilization: 0.55 → 0.45 (avoid CUDA UVM spilling) - Updated critic model path and experiment name Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index 9c0d68f..1e7d7bf 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -1,5 +1,5 @@ # SDPO config for sparse parity challenge on 1x A100 80GB (a100-backup-1) -# Adapted from autoresearch_sdpo.yaml for single GPU + shorter context +# Qwen3-8B + LoRA r32, LR 5e-5, 4x parallel tool calls, 0.45 GPU mem util # Usage: bash scripts/run_sparse_parity.sh [experiment-name] defaults: @@ -22,7 +22,7 @@ data: actor_rollout_ref: model: - path: Qwen/Qwen3-14B + path: Qwen/Qwen3-8B trust_remote_code: True lora_rank: 32 lora_alpha: 16 @@ -66,14 +66,14 @@ actor_rollout_ref: optim: optimizer: AdamW optimizer_impl: torch.optim - lr: 1e-6 + lr: 5e-5 lr_warmup_steps: 5 rollout: name: vllm n: 1 temperature: 1.0 tensor_model_parallel_size: 1 - gpu_memory_utilization: 0.55 + gpu_memory_utilization: 0.45 max_model_len: ${max_model_len} max_num_batched_tokens: ${max_model_len} calculate_log_probs: True @@ -88,7 +88,7 @@ actor_rollout_ref: enable: True max_user_turns: 10 max_assistant_turns: 10 - max_parallel_calls: 1 + max_parallel_calls: 4 max_tool_response_length: 10000 tool_response_truncate_side: right tool_config_path: ${vars.dir}/configs/sparse_parity_bash_tool.yaml @@ -119,7 +119,7 @@ critic: strategy: fsdp2 ulysses_sequence_parallel_size: 1 model: - path: Qwen/Qwen3-14B + path: Qwen/Qwen3-8B custom_reward_function: path: ${vars.dir}/training/reward.py @@ -130,7 +130,7 @@ reward_model: trainer: project_name: sparse-parity-sdpo group_name: sparse-parity-sdpo - experiment_name: qwen3-14b-sdpo-lora32 + experiment_name: qwen3-8b-sdpo-lora32 n_gpus_per_node: 1 nnodes: 1 total_epochs: 1000 From 95d54c69575001e15d3a722b2766b6c929c48d63 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 10 Apr 2026 10:19:41 -0700 Subject: [PATCH 02/10] Fix max_model_len for Qwen3-8B (40960 max_position_embeddings) VERL's vllm_async_server overwrites config max_model_len with the HF model's max_position_embeddings. Qwen3-8B has 40960 (vs 32768 for 14B). Actual sequence lengths are still bounded by max_prompt+max_response=32768. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index 1e7d7bf..1a26d16 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -6,7 +6,7 @@ defaults: - sdpo - _self_ -max_model_len: 32768 +max_model_len: 40960 # Qwen3-8B max_position_embeddings; actual seqs bounded by prompt+response data: train_files: ["${vars.dir}/data/sparse_parity/train.parquet"] @@ -83,7 +83,7 @@ actor_rollout_ref: rope_scaling: rope_type: yarn factor: 1.0 - original_max_position_embeddings: 32768 + original_max_position_embeddings: 40960 multi_turn: enable: True max_user_turns: 10 From 8a9278f7f2070ebfc3349263687da543ff108d67 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 10 Apr 2026 10:22:56 -0700 Subject: [PATCH 03/10] =?UTF-8?q?Remove=20rope=5Fscaling=20override=20?= =?UTF-8?q?=E2=80=94=20Qwen3-8B=20handles=20rope=20natively?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hf_overrides rope_scaling was from 14B config and caused vLLM to derive max_position_embeddings=32768 (from original_max_position_embeddings), conflicting with the 40960 passed as max_model_len. Without the override, vLLM uses the model's native 40960 and our 32768 max_model_len fits fine. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index 1a26d16..33a46e8 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -6,7 +6,7 @@ defaults: - sdpo - _self_ -max_model_len: 40960 # Qwen3-8B max_position_embeddings; actual seqs bounded by prompt+response +max_model_len: 32768 data: train_files: ["${vars.dir}/data/sparse_parity/train.parquet"] @@ -77,13 +77,6 @@ actor_rollout_ref: max_model_len: ${max_model_len} max_num_batched_tokens: ${max_model_len} calculate_log_probs: True - engine_kwargs: - vllm: - hf_overrides: - rope_scaling: - rope_type: yarn - factor: 1.0 - original_max_position_embeddings: 40960 multi_turn: enable: True max_user_turns: 10 From 337e7ddb64ea9dfaf0d1d89d88cf9942bede162c Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 10 Apr 2026 15:01:04 -0700 Subject: [PATCH 04/10] =?UTF-8?q?Reduce=20gpu=5Fmemory=5Futilization=20to?= =?UTF-8?q?=200.35=20=E2=80=94=20OOM=20at=200.45=20during=20actor=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 OOM: 97GB allocated by PyTorch on 80GB A100. vLLM 0.45 × 80 = 36GB leaves insufficient headroom for model weights + optimizer + activations. 0.35 × 80 = 28GB for vLLM, leaving ~52GB for training. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index 33a46e8..d7ff156 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -73,7 +73,7 @@ actor_rollout_ref: n: 1 temperature: 1.0 tensor_model_parallel_size: 1 - gpu_memory_utilization: 0.45 + gpu_memory_utilization: 0.35 max_model_len: ${max_model_len} max_num_batched_tokens: ${max_model_len} calculate_log_probs: True From ee79a0855a22eb51443d170b99f1fd5124e4428c Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 10 Apr 2026 16:33:40 -0700 Subject: [PATCH 05/10] Enable gradient checkpointing to fix OOM during actor update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Still OOM at gpu_memory_utilization=0.35 — tried to allocate 14.38 GiB during backward pass. Gradient checkpointing recomputes activations instead of storing them, trading ~30% compute for significant memory savings. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index d7ff156..b935881 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -27,6 +27,7 @@ actor_rollout_ref: lora_rank: 32 lora_alpha: 16 target_modules: all-linear + enable_gradient_checkpointing: true override_config: attn_implementation: flash_attention_2 actor: From 0cb6d075eb32d5121f47ef9b87c286eb4a5e808d Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Sat, 11 Apr 2026 19:13:50 -0700 Subject: [PATCH 06/10] Reduce ppo_max_token_len_per_gpu to 16384 to fix logits OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OOM at logsumexp: 40960 × 151936 vocab × 4 bytes = ~23 GB logits tensor. 16384 tokens → ~9.4 GB logits, fits within remaining GPU memory after vLLM (28 GB) + model weights (16 GB). Sequences longer than 16384 will be split across micro-batches automatically. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index b935881..a2b2b54 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -33,7 +33,7 @@ actor_rollout_ref: actor: ppo_micro_batch_size_per_gpu: 1 ppo_mini_batch_size: 8 - ppo_max_token_len_per_gpu: ${max_model_len} + ppo_max_token_len_per_gpu: 16384 use_dynamic_bsz: False ulysses_sequence_parallel_size: 1 strategy: fsdp2 From 5e86c14aac699f2cb01bc69186b6ebf7a0cdc5e7 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Sun, 12 Apr 2026 11:06:35 -0700 Subject: [PATCH 07/10] =?UTF-8?q?Lower=20LR=20to=203e-5=20=E2=80=94=20poli?= =?UTF-8?q?cy=20collapsed=20at=205e-5=20after=20warmup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 4 (LR 3e-5) hit DMC 95,741 — big improvement from 637K baseline. Steps 5-6 (LR 4e-5, 5e-5) collapsed to 999M DMC with zero rewards. 3e-5 appears to be the sweet spot for LoRA r32 on 8B model. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index a2b2b54..e74e273 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -67,7 +67,7 @@ actor_rollout_ref: optim: optimizer: AdamW optimizer_impl: torch.optim - lr: 5e-5 + lr: 3e-5 lr_warmup_steps: 5 rollout: name: vllm From 0db534b6d71b806be0390a956d500c1e8a0e3617 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 13 Apr 2026 10:46:44 -0700 Subject: [PATCH 08/10] =?UTF-8?q?Halve=20context=20to=2016K=20to=20fix=20l?= =?UTF-8?q?ogits=20OOM=20=E2=80=94=20ppo=5Fmax=5Ftoken=5Flen=20follows=20m?= =?UTF-8?q?ax=5Fmodel=5Flen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ppo_max_token_len_per_gpu resolves from ${max_model_len} in base config and CLI overrides are silently ignored (Hydra structured config issue). Reducing max_model_len from 32K to 16K halves the logits tensor: 16384 × 151936 × 4 = ~9.4 GB (was ~18.6 GB at 32K). Context budget: 4K prompt + 12K response = 16K. Sparse parity solutions are ~100 lines of numpy — 12K response is more than sufficient. Co-Authored-By: Claude Opus 4.6 --- configs/sparse_parity_sdpo.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/sparse_parity_sdpo.yaml b/configs/sparse_parity_sdpo.yaml index e74e273..483a8dc 100644 --- a/configs/sparse_parity_sdpo.yaml +++ b/configs/sparse_parity_sdpo.yaml @@ -6,14 +6,14 @@ defaults: - sdpo - _self_ -max_model_len: 32768 +max_model_len: 16384 data: train_files: ["${vars.dir}/data/sparse_parity/train.parquet"] val_files: ["${vars.dir}/data/sparse_parity/test.parquet"] train_batch_size: 8 - max_prompt_length: 8192 - max_response_length: 24576 + max_prompt_length: 4096 + max_response_length: 12288 apply_chat_template_kwargs: {"enable_thinking": true} return_raw_chat: True filter_overlong_prompts: True @@ -46,7 +46,7 @@ actor_rollout_ref: loss_mode: sdpo self_distillation: teacher_update_rate: 0.01 - max_reprompt_len: 8192 + max_reprompt_len: 4096 is_clip: 2.0 alpha: 1.0 distillation_topk: 20 From 809534946ecdd4c933e3585b05416f5c5217563d Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 13 Apr 2026 10:50:21 -0700 Subject: [PATCH 09/10] Force ppo_max_token_len_per_gpu from max_model_len at runtime Hydra structured config silently ignores YAML and CLI overrides for ppo_max_token_len_per_gpu (always resolves to 32768 from base config interpolation). Force it programmatically after config resolution. Co-Authored-By: Claude Opus 4.6 --- training/run_sdpo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/training/run_sdpo.py b/training/run_sdpo.py index 97bb76d..2e95562 100644 --- a/training/run_sdpo.py +++ b/training/run_sdpo.py @@ -102,6 +102,11 @@ def run(self, config): def main(config): from verl.utils.device import auto_set_device auto_set_device(config) + # Force ppo_max_token_len_per_gpu to match max_model_len — Hydra structured + # config silently ignores YAML/CLI overrides for this interpolated field. + from omegaconf import OmegaConf + with OmegaConf.read_write(config): + config.actor_rollout_ref.actor.ppo_max_token_len_per_gpu = config.max_model_len task_runner_class = ray.remote(num_cpus=1)(PatchedTaskRunner) run_ppo(config, task_runner_class=task_runner_class) From 330fce374c91f67527e9a57845c1ec9cc1c720e3 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 13 Apr 2026 10:52:15 -0700 Subject: [PATCH 10/10] Fix OmegaConf API: use flag_override instead of read_write Older omegaconf version doesn't have read_write context manager. Co-Authored-By: Claude Opus 4.6 --- training/run_sdpo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/training/run_sdpo.py b/training/run_sdpo.py index 2e95562..70cf0f1 100644 --- a/training/run_sdpo.py +++ b/training/run_sdpo.py @@ -104,8 +104,8 @@ def main(config): auto_set_device(config) # Force ppo_max_token_len_per_gpu to match max_model_len — Hydra structured # config silently ignores YAML/CLI overrides for this interpolated field. - from omegaconf import OmegaConf - with OmegaConf.read_write(config): + from omegaconf import OmegaConf, flag_override + with flag_override(config, "struct", False): config.actor_rollout_ref.actor.ppo_max_token_len_per_gpu = config.max_model_len task_runner_class = ray.remote(num_cpus=1)(PatchedTaskRunner) run_ppo(config, task_runner_class=task_runner_class)