AbstractPhil commited on
Commit
222eb5d
·
verified ·
1 Parent(s): 382db9c

bed: alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16 weights

Browse files
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/addr_head.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04b2578e012ca48e8a0bbf5c5e353834f53518be5dbb08e8b3dc145be3b470d8
3
+ size 16584
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "bed-alhena-both",
3
+ "d_model": 256,
4
+ "n_layers": 6,
5
+ "n_heads": 8,
6
+ "context": 4096,
7
+ "vocab_size": 256,
8
+ "tokenizer": "byte-trigram",
9
+ "hub_layers": [],
10
+ "hub_K": 32,
11
+ "hub_D": 16,
12
+ "tau": 0.1,
13
+ "bank_experts": 7,
14
+ "bank_ff": 256,
15
+ "head_K": 64,
16
+ "head_D": 16,
17
+ "head_mode": "",
18
+ "head_dial_band": 4.0,
19
+ "head_dial_rho": 0.015625,
20
+ "gate_init": -3.0,
21
+ "tie_embeddings": false,
22
+ "hub_chunk": 128,
23
+ "hub_const": 1,
24
+ "hub_ckpt": 0,
25
+ "fusion": null,
26
+ "qk_norm": "",
27
+ "attn_fp32": false,
28
+ "attn_kernel": "sdpa",
29
+ "lookup_arm_sites": [],
30
+ "lookup_arm_depth": 1,
31
+ "attn_rotary": true,
32
+ "hub_rotary": false,
33
+ "rotary_base": 10000.0,
34
+ "hub_rotary_frac": 1.0,
35
+ "hub_dual": false,
36
+ "pos_table": true,
37
+ "bank_byte_table": "add",
38
+ "bank_last_fix": "",
39
+ "fix_rho": 0.0625,
40
+ "fix_warm": 200,
41
+ "fix_clamp": [
42
+ 0.25,
43
+ 4.0
44
+ ],
45
+ "canon": ""
46
+ }
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/key_rot.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0d0beb49e4ad077b36d3eb5a3fdb41767146904500db46503b528ae751491e1
3
+ size 147536
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c830a92ba9e37923b4b434a27425647949daf643ada75d41882d347f5fbf0b8
3
+ size 36916028
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/scale.json ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "grid": "ag",
3
+ "grid_p": 0.5,
4
+ "n_ag": 4,
5
+ "rotary": {
6
+ "form": "anchored",
7
+ "pairs": 16,
8
+ "rel_pairs": 4,
9
+ "content_pairs": 4,
10
+ "anchor_freqs": 4,
11
+ "levels": 2,
12
+ "ntk_scale": 4.0,
13
+ "base": 10000.0,
14
+ "null": 8,
15
+ "w": [
16
+ 1.0,
17
+ 0.06299605249474366,
18
+ 0.003968502629920499,
19
+ 0.00025,
20
+ 0.0,
21
+ 0.0,
22
+ 0.0,
23
+ 0.0,
24
+ 0.6981317007977318,
25
+ 1.3962634015954636,
26
+ 2.0943951023931953,
27
+ 2.792526803190927,
28
+ 0.6981317007977318,
29
+ 1.3962634015954636,
30
+ 2.0943951023931953,
31
+ 2.792526803190927
32
+ ],
33
+ "kind": [
34
+ 0,
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 2,
43
+ 2,
44
+ 2,
45
+ 2,
46
+ 3,
47
+ 3,
48
+ 3,
49
+ 3
50
+ ]
51
+ },
52
+ "head": "digit",
53
+ "head_lam": 1.0,
54
+ "layout": {
55
+ "slot": 64,
56
+ "comps": 48,
57
+ "radix": 8,
58
+ "levels": 2
59
+ },
60
+ "grid_layout": "AG48",
61
+ "grid_hard": 0.5,
62
+ "value_addr": {
63
+ "levels": 2,
64
+ "digits": 8,
65
+ "d": 256
66
+ },
67
+ "key_rot": {
68
+ "pairs": 16,
69
+ "k": 9,
70
+ "vocab": 256,
71
+ "block": 4,
72
+ "head": 0,
73
+ "seed_tag": "KEYROT|v1|j16",
74
+ "norm": 2,
75
+ "rho": 2.18
76
+ },
77
+ "ptr": {
78
+ "lam": 0.1,
79
+ "block": 4,
80
+ "head": 0
81
+ }
82
+ }
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/value_addr.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e25fe9e3aa63a279defa4652f6ab23979659bb2effe203eccfa56e23983b5b2a
3
+ size 16464