| # Laptop smoke-test model (~7.4M params, ~20M tokens on CPU). | |
| name: tiny | |
| model: | |
| d_model: 256 | |
| n_layers: 4 | |
| n_heads: 4 | |
| ffn_hidden: 704 | |
| context_len: 512 | |
| train: | |
| peak_lr: 1.0e-3 | |
| total_steps: 2440 | |
| micro_batch_size: 8 | |
| grad_accum: 4 | |
| precision: fp32 | |
| data_dir: data/processed | |
| mix: {astro: 0.30, arxiv: 0.15, textbook: 0.05, general: 0.50} | |
| max_epochs: {textbook: 3, astro: 2} | |