{
 "format": "monarch-latent-math/1",
 "name": "Monarch Latent Math 7B",
 "base_model": "allenai/OLMoE-1B-7B-0125-Instruct",
 "base_revision": "b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e",
 "num_layers": 16,
 "hidden_size": 2048,
 "lora": {
  "targets": [
   "q_proj",
   "k_proj",
   "v_proj",
   "o_proj"
  ],
  "r": 128,
  "alpha": 32.0
 },
 "moe_adapter": {
  "r": 128,
  "alpha": 32.0
 },
 "projection": "Dropout, Linear(d, d), GELU, Linear(d, d), LayerNorm(d)",
 "latent_routing": "lock",
 "special_tokens": {
  "<bot>": 0,
  "<eot>": 1,
  "<pause>": 2,
  "note": "ids are base vocabulary size + offset (base tokenizer.json with added tokens)"
 },
 "tokenizer": {
  "file": "tokenizer.json",
  "sha256": "d1e645ebd850d79567e531a3c103ac575d8e9cf45fa941420afc584b293438ea",
  "base_size": 50281
 },
 "prompt": {
  "bos": false,
  "template": "raw question, then <bot>"
 },
 "default_steps": 6,
 "tensors_dtype": "float32"
}