{
  "model_type": "vbq",
  "architectures": [
    "VBQ"
  ],
  "model_cfg": {
    "vocab_size": 32768,
    "d_model": 640,
    "n_layers": 18,
    "n_heads": 10,
    "n_kv_heads": 5,
    "mlp_hidden": 1664,
    "max_seq_len": 2048,
    "rope_theta": 10000.0,
    "norm_eps": 1e-05,
    "qk_norm": true,
    "tie_embeddings": true,
    "n_experts": 0,
    "capacity_factor": 1.25
  },
  "train_step": 8000,
  "total_steps": null,
  "display_name": "MP-01 100M base",
  "stage": "cooldown from expansion step 19000; 8,000 steps of 262,144 tokens (2.10B) on a second corpus",
  "lineage": [
    "EB-75M base (4.0B tokens)",
    "grown to 100M (0.61B)",
    "chat fine-tune (0.18B)",
    "expansion (to step 19000)",
    "cooldown (8000 steps)"
  ]
}