| { | |
| "name": "mini-beatrix-3", | |
| "d_model": 1024, | |
| "n_layers": 32, | |
| "n_heads": 16, | |
| "context": 4096, | |
| "vocab_size": 256, | |
| "tokenizer": "byte-trigram", | |
| "hub_layers": [ | |
| 0, | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9, | |
| 10, | |
| 11, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 16, | |
| 17, | |
| 18, | |
| 19, | |
| 20, | |
| 21, | |
| 22, | |
| 23, | |
| 24, | |
| 25, | |
| 26, | |
| 27, | |
| 28, | |
| 29, | |
| 30, | |
| 31 | |
| ], | |
| "hub_K": 64, | |
| "hub_D": 128, | |
| "tau": 0.1, | |
| "bank_experts": 3, | |
| "bank_ff": 1024, | |
| "head_K": 256, | |
| "head_D": 256, | |
| "gate_init": -3.0, | |
| "tie_embeddings": false, | |
| "hub_chunk": 256, | |
| "hub_const": 4, | |
| "hub_ckpt": 0, | |
| "model_type": "mini-beatrix", | |
| "architectures": [ | |
| "MiniBeatrixForCausalLM" | |
| ], | |
| "auto_map": { | |
| "AutoConfig": "modeling_minibeatrix.MiniBeatrixConfig", | |
| "AutoModelForCausalLM": "modeling_minibeatrix.MiniBeatrixForCausalLM" | |
| }, | |
| "torch_dtype": "float32", | |
| "use_cache": false, | |
| "trunk": { | |
| "craft": "mini-beatrix-3", | |
| "step": 245674, | |
| "base_model_id": "alephllm/mini-beatrix-3@step245674", | |
| "note": "the core weights are the same with arms off and arms on; the arms are separate files" | |
| }, | |
| "default_arm": null | |
| } |