| { | |
| "model_type": "vbq", | |
| "architectures": [ | |
| "VBQ" | |
| ], | |
| "model_cfg": { | |
| "vocab_size": 32768, | |
| "d_model": 640, | |
| "n_layers": 18, | |
| "n_heads": 10, | |
| "n_kv_heads": 5, | |
| "mlp_hidden": 1664, | |
| "max_seq_len": 2048, | |
| "rope_theta": 10000.0, | |
| "norm_eps": 1e-05, | |
| "qk_norm": true, | |
| "tie_embeddings": true, | |
| "n_experts": 0, | |
| "capacity_factor": 1.25 | |
| }, | |
| "train_step": 8000, | |
| "total_steps": null, | |
| "display_name": "MP-01 100M base", | |
| "stage": "cooldown from expansion step 19000; 8,000 steps of 262,144 tokens (2.10B) on a second corpus", | |
| "lineage": [ | |
| "EB-75M base (4.0B tokens)", | |
| "grown to 100M (0.61B)", | |
| "chat fine-tune (0.18B)", | |
| "expansion (to step 19000)", | |
| "cooldown (8000 steps)" | |
| ] | |
| } |