hyperparameters.json

{
  "FFN variant": "4 \u2014 4 experts, 2 active per token (1 shared + 3 routed), total-param matched",
  "params (total / active)": "8.8M / 7.2M",
  "FFN hidden width": "256 per expert",
  "MoE aux weight": "0.01",
  "d_model / n_layers / n_heads": "256 / 6 / 8",
  "context length": "256",
  "positional encoding": "RoPE (base 10000)",
  "tokenizer": "sentencepiece unigram, vocab 16000, character coverage 0.9995, byte fallback",
  "optimizer": "AdamW(betas=(0.9, 0.95), eps=1e-8)",
  "learning rate": "0.0003",
  "lr schedule": "linear warmup 1740 steps, cosine to 0.1x",
  "weight decay": "0.1 (matrices only)",
  "gradient clip": "1.0",
  "batch size": "32 x 1 accum",
  "dropout": "0.1",
  "precision": "fp16",
  "token budget": "120,000,000 (120,000,979 seen)",
  "seed": "26"
}