| { |
| "FFN variant": "5 \u2014 4 experts, 2 active per token, active-param matched", |
| "params (total / active)": "12.0M / 8.8M", |
| "FFN hidden width": "512 per expert", |
| "MoE aux weight": "0.01", |
| "d_model / n_layers / n_heads": "256 / 6 / 8", |
| "context length": "256", |
| "positional encoding": "RoPE (base 10000)", |
| "tokenizer": "sentencepiece unigram, vocab 16000, character coverage 0.9995, byte fallback", |
| "optimizer": "AdamW(betas=(0.9, 0.95), eps=1e-8)", |
| "learning rate": "0.0003", |
| "lr schedule": "linear warmup 1740 steps, cosine to 0.1x", |
| "weight decay": "0.1 (matrices only)", |
| "gradient clip": "1.0", |
| "batch size": "32 x 1 accum", |
| "dropout": "0.1", |
| "precision": "fp16", |
| "token budget": "120,000,000 (120,001,089 seen)", |
| "seed": "26" |
| } |
