| # Phase‑2 async configuration (disaggregated rollout & trainer) | |
| model: | |
| name: "Qwen2.5-0.5B" | |
| path: "${MODEL_PATH}" # fill via env var or replace manually | |
| training: | |
| epochs: 3 | |
| batch_size: 64 | |
| learning_rate: 5e-6 | |
| optimizer: "adamw" | |
| loss: "ppo" # or "grpo" | |
| clip_eps: 0.2 | |
| use_m2po: true | |
| rollout: | |
| backend: "vllm" # options: "vllm" or "sglang" | |
| max_new_tokens: 256 | |
| temperature: 0.7 | |
| top_p: 0.95 | |
| batch_size: 8 # prompts per async generation batch | |
| replay_buffer: | |
| max_size: 20000 | |
| staleness_bound: 8 | |
| orchestrator: | |
| sync_interval: 10 # steps between weight syncs | |
| num_rollout_workers: 4 | |
| num_trainer_workers: 2 | |