File size: 1,078 Bytes
9494036 acf94d3 9494036 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | {
"name": "mini-beatrix-3",
"d_model": 1024,
"n_layers": 32,
"n_heads": 16,
"context": 4096,
"vocab_size": 256,
"tokenizer": "byte-trigram",
"hub_layers": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31
],
"hub_K": 64,
"hub_D": 128,
"tau": 0.1,
"bank_experts": 3,
"bank_ff": 1024,
"head_K": 256,
"head_D": 256,
"gate_init": -3.0,
"tie_embeddings": false,
"hub_chunk": 256,
"hub_const": 4,
"hub_ckpt": 0,
"model_type": "mini-beatrix",
"architectures": [
"MiniBeatrixForCausalLM"
],
"auto_map": {
"AutoConfig": "modeling_minibeatrix.MiniBeatrixConfig",
"AutoModelForCausalLM": "modeling_minibeatrix.MiniBeatrixForCausalLM"
},
"torch_dtype": "float32",
"use_cache": false,
"trunk": {
"craft": "mini-beatrix-3",
"step": 245674,
"base_model_id": "alephllm/mini-beatrix-3@step245674",
"note": "the core weights are the same with arms off and arms on; the arms are separate files"
},
"default_arm": null
} |