| { |
| "model_type": "vbq", |
| "architectures": [ |
| "VBQ" |
| ], |
| "model_cfg": { |
| "vocab_size": 32768, |
| "d_model": 640, |
| "n_layers": 18, |
| "n_heads": 10, |
| "n_kv_heads": 5, |
| "mlp_hidden": 1664, |
| "max_seq_len": 2048, |
| "rope_theta": 10000.0, |
| "norm_eps": 1e-05, |
| "qk_norm": true, |
| "tie_embeddings": true, |
| "n_experts": 0, |
| "capacity_factor": 1.25 |
| }, |
| "train_step": 8000, |
| "total_steps": null, |
| "display_name": "MP-01 100M Preview", |
| "stage": "cooldown from expansion step 19000; 8,000 steps of 262,144 tokens (2.10B) on a second corpus", |
| "lineage": [ |
| "EB-75M base (4.0B tokens)", |
| "grown to 100M (0.61B)", |
| "chat fine-tune (0.18B)", |
| "expansion (to step 19000)", |
| "cooldown (8000 steps)" |
| ] |
| } |
