bed: alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16 weights
Browse files- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/addr_head.safetensors +3 -0
- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/config.json +46 -0
- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/key_rot.safetensors +3 -0
- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/model.safetensors +3 -0
- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/scale.json +82 -0
- bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/value_addr.safetensors +3 -0
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/addr_head.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:04b2578e012ca48e8a0bbf5c5e353834f53518be5dbb08e8b3dc145be3b470d8
|
| 3 |
+
size 16584
|
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/config.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "bed-alhena-both",
|
| 3 |
+
"d_model": 256,
|
| 4 |
+
"n_layers": 6,
|
| 5 |
+
"n_heads": 8,
|
| 6 |
+
"context": 4096,
|
| 7 |
+
"vocab_size": 256,
|
| 8 |
+
"tokenizer": "byte-trigram",
|
| 9 |
+
"hub_layers": [],
|
| 10 |
+
"hub_K": 32,
|
| 11 |
+
"hub_D": 16,
|
| 12 |
+
"tau": 0.1,
|
| 13 |
+
"bank_experts": 7,
|
| 14 |
+
"bank_ff": 256,
|
| 15 |
+
"head_K": 64,
|
| 16 |
+
"head_D": 16,
|
| 17 |
+
"head_mode": "",
|
| 18 |
+
"head_dial_band": 4.0,
|
| 19 |
+
"head_dial_rho": 0.015625,
|
| 20 |
+
"gate_init": -3.0,
|
| 21 |
+
"tie_embeddings": false,
|
| 22 |
+
"hub_chunk": 128,
|
| 23 |
+
"hub_const": 1,
|
| 24 |
+
"hub_ckpt": 0,
|
| 25 |
+
"fusion": null,
|
| 26 |
+
"qk_norm": "",
|
| 27 |
+
"attn_fp32": false,
|
| 28 |
+
"attn_kernel": "sdpa",
|
| 29 |
+
"lookup_arm_sites": [],
|
| 30 |
+
"lookup_arm_depth": 1,
|
| 31 |
+
"attn_rotary": true,
|
| 32 |
+
"hub_rotary": false,
|
| 33 |
+
"rotary_base": 10000.0,
|
| 34 |
+
"hub_rotary_frac": 1.0,
|
| 35 |
+
"hub_dual": false,
|
| 36 |
+
"pos_table": true,
|
| 37 |
+
"bank_byte_table": "add",
|
| 38 |
+
"bank_last_fix": "",
|
| 39 |
+
"fix_rho": 0.0625,
|
| 40 |
+
"fix_warm": 200,
|
| 41 |
+
"fix_clamp": [
|
| 42 |
+
0.25,
|
| 43 |
+
4.0
|
| 44 |
+
],
|
| 45 |
+
"canon": ""
|
| 46 |
+
}
|
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/key_rot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b0d0beb49e4ad077b36d3eb5a3fdb41767146904500db46503b528ae751491e1
|
| 3 |
+
size 147536
|
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8c830a92ba9e37923b4b434a27425647949daf643ada75d41882d347f5fbf0b8
|
| 3 |
+
size 36916028
|
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/scale.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"grid": "ag",
|
| 3 |
+
"grid_p": 0.5,
|
| 4 |
+
"n_ag": 4,
|
| 5 |
+
"rotary": {
|
| 6 |
+
"form": "anchored",
|
| 7 |
+
"pairs": 16,
|
| 8 |
+
"rel_pairs": 4,
|
| 9 |
+
"content_pairs": 4,
|
| 10 |
+
"anchor_freqs": 4,
|
| 11 |
+
"levels": 2,
|
| 12 |
+
"ntk_scale": 4.0,
|
| 13 |
+
"base": 10000.0,
|
| 14 |
+
"null": 8,
|
| 15 |
+
"w": [
|
| 16 |
+
1.0,
|
| 17 |
+
0.06299605249474366,
|
| 18 |
+
0.003968502629920499,
|
| 19 |
+
0.00025,
|
| 20 |
+
0.0,
|
| 21 |
+
0.0,
|
| 22 |
+
0.0,
|
| 23 |
+
0.0,
|
| 24 |
+
0.6981317007977318,
|
| 25 |
+
1.3962634015954636,
|
| 26 |
+
2.0943951023931953,
|
| 27 |
+
2.792526803190927,
|
| 28 |
+
0.6981317007977318,
|
| 29 |
+
1.3962634015954636,
|
| 30 |
+
2.0943951023931953,
|
| 31 |
+
2.792526803190927
|
| 32 |
+
],
|
| 33 |
+
"kind": [
|
| 34 |
+
0,
|
| 35 |
+
0,
|
| 36 |
+
0,
|
| 37 |
+
0,
|
| 38 |
+
1,
|
| 39 |
+
1,
|
| 40 |
+
1,
|
| 41 |
+
1,
|
| 42 |
+
2,
|
| 43 |
+
2,
|
| 44 |
+
2,
|
| 45 |
+
2,
|
| 46 |
+
3,
|
| 47 |
+
3,
|
| 48 |
+
3,
|
| 49 |
+
3
|
| 50 |
+
]
|
| 51 |
+
},
|
| 52 |
+
"head": "digit",
|
| 53 |
+
"head_lam": 1.0,
|
| 54 |
+
"layout": {
|
| 55 |
+
"slot": 64,
|
| 56 |
+
"comps": 48,
|
| 57 |
+
"radix": 8,
|
| 58 |
+
"levels": 2
|
| 59 |
+
},
|
| 60 |
+
"grid_layout": "AG48",
|
| 61 |
+
"grid_hard": 0.5,
|
| 62 |
+
"value_addr": {
|
| 63 |
+
"levels": 2,
|
| 64 |
+
"digits": 8,
|
| 65 |
+
"d": 256
|
| 66 |
+
},
|
| 67 |
+
"key_rot": {
|
| 68 |
+
"pairs": 16,
|
| 69 |
+
"k": 9,
|
| 70 |
+
"vocab": 256,
|
| 71 |
+
"block": 4,
|
| 72 |
+
"head": 0,
|
| 73 |
+
"seed_tag": "KEYROT|v1|j16",
|
| 74 |
+
"norm": 2,
|
| 75 |
+
"rho": 2.18
|
| 76 |
+
},
|
| 77 |
+
"ptr": {
|
| 78 |
+
"lam": 0.1,
|
| 79 |
+
"block": 4,
|
| 80 |
+
"head": 0
|
| 81 |
+
}
|
| 82 |
+
}
|
bed/weights/alhena-both_s6_L24000_ctx4096_frdyn_cap_mode_sz32-128-256-512_r4_n30_o0_askany_gag50_cp4_gc48_kh50_va1_pt10b4h0_krt16r218_ahdigit1_rec16/value_addr.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e25fe9e3aa63a279defa4652f6ab23979659bb2effe203eccfa56e23983b5b2a
|
| 3 |
+
size 16464
|