{
  "architectures": [
    "PhobosScratchLM"
  ],
  "model_type": "phobos_cortical_moe",
  "organization": "MarsNext",
  "vocab_size": 32768,
  "d_model": 512,
  "num_layers": 8,
  "num_heads": 8,
  "num_kv_heads": 4,
  "max_seq_len": 2048,
  "shared_mlp_neurons": 1024,
  "leaf_mlp_neurons": 256,
  "num_branches": 4,
  "leaves_per_layer": [
    16,
    16,
    16,
    16,
    16,
    16,
    16,
    16
  ],
  "top_k_leaves": 2,
  "total_parameters": 96256569,
  "active_parameters_per_token": 52216377,
  "torch_dtype": "bfloat16"
}