| { |
| "FFN variant": "4 \u2014 4 experts, 2 active per token (1 shared + 3 routed), total-param matched", |
| "params (total / active)": "8.8M / 7.2M", |
| "FFN hidden width": "256 per expert", |
| "MoE aux weight": "0.01", |
| "d_model / n_layers / n_heads": "256 / 6 / 8", |
| "context length": "256", |
| "positional encoding": "RoPE (base 10000)", |
| "tokenizer": "sentencepiece unigram, vocab 16000, character coverage 0.9995, byte fallback", |
| "optimizer": "AdamW(betas=(0.9, 0.95), eps=1e-8)", |
| "learning rate": "0.0003", |
| "lr schedule": "linear warmup 1740 steps, cosine to 0.1x", |
| "weight decay": "0.1 (matrices only)", |
| "gradient clip": "1.0", |
| "batch size": "32 x 1 accum", |
| "dropout": "0.1", |
| "precision": "fp16", |
| "token budget": "120,000,000 (120,000,979 seen)", |
| "seed": "26" |
| } |
