| { | |
| "format": "monarch-latent-math/1", | |
| "name": "Monarch Latent Math 7B", | |
| "base_model": "allenai/OLMoE-1B-7B-0125-Instruct", | |
| "base_revision": "b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e", | |
| "num_layers": 16, | |
| "hidden_size": 2048, | |
| "lora": { | |
| "targets": [ | |
| "q_proj", | |
| "k_proj", | |
| "v_proj", | |
| "o_proj" | |
| ], | |
| "r": 128, | |
| "alpha": 32.0 | |
| }, | |
| "moe_adapter": { | |
| "r": 128, | |
| "alpha": 32.0 | |
| }, | |
| "projection": "Dropout, Linear(d, d), GELU, Linear(d, d), LayerNorm(d)", | |
| "latent_routing": "lock", | |
| "special_tokens": { | |
| "<bot>": 0, | |
| "<eot>": 1, | |
| "<pause>": 2, | |
| "note": "ids are base vocabulary size + offset (base tokenizer.json with added tokens)" | |
| }, | |
| "tokenizer": { | |
| "file": "tokenizer.json", | |
| "sha256": "d1e645ebd850d79567e531a3c103ac575d8e9cf45fa941420afc584b293438ea", | |
| "base_size": 50281 | |
| }, | |
| "prompt": { | |
| "bos": false, | |
| "template": "raw question, then <bot>" | |
| }, | |
| "default_steps": 6, | |
| "tensors_dtype": "float32" | |
| } | |