| { |
| "metadata": { |
| "ParamSize": 483, |
| "ParamBytes": 438221680.0, |
| "BitsPerParam": 4.501606609864174 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 32165760, |
| "records": [ |
| { |
| "name": "model.embed_tokens.q_weight", |
| "shape": [ |
| 50259, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 32165760, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "0237824afdd0afc36fe1ef7599975ef2" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 31592688, |
| "records": [ |
| { |
| "name": "model.embed_tokens.q_scale", |
| "shape": [ |
| 50259, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 4020720, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.0.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 4020720 |
| }, |
| { |
| "name": "model.layers.0.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 4023280 |
| }, |
| { |
| "name": "model.layers.0.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 5252080 |
| }, |
| { |
| "name": "model.layers.0.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 5405680 |
| }, |
| { |
| "name": "model.layers.0.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 5405808 |
| }, |
| { |
| "name": "model.layers.0.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 5405936 |
| }, |
| { |
| "name": "model.layers.0.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 6225136 |
| }, |
| { |
| "name": "model.layers.0.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 6327536 |
| }, |
| { |
| "name": "model.layers.0.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 6330096 |
| }, |
| { |
| "name": "model.layers.0.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 10917616 |
| }, |
| { |
| "name": "model.layers.0.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 11491056 |
| }, |
| { |
| "name": "model.layers.0.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 13784816 |
| }, |
| { |
| "name": "model.layers.1.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 14071536 |
| }, |
| { |
| "name": "model.layers.1.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 14074096 |
| }, |
| { |
| "name": "model.layers.1.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 15302896 |
| }, |
| { |
| "name": "model.layers.1.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 15456496 |
| }, |
| { |
| "name": "model.layers.1.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 15456624 |
| }, |
| { |
| "name": "model.layers.1.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 15456752 |
| }, |
| { |
| "name": "model.layers.1.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 16275952 |
| }, |
| { |
| "name": "model.layers.1.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 16378352 |
| }, |
| { |
| "name": "model.layers.1.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 16380912 |
| }, |
| { |
| "name": "model.layers.1.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 20968432 |
| }, |
| { |
| "name": "model.layers.1.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 21541872 |
| }, |
| { |
| "name": "model.layers.1.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 23835632 |
| }, |
| { |
| "name": "model.layers.2.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 24122352 |
| }, |
| { |
| "name": "model.layers.2.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 24124912 |
| }, |
| { |
| "name": "model.layers.2.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 25353712 |
| }, |
| { |
| "name": "model.layers.2.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 25507312 |
| }, |
| { |
| "name": "model.layers.2.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 25507440 |
| }, |
| { |
| "name": "model.layers.2.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 25507568 |
| }, |
| { |
| "name": "model.layers.2.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 26326768 |
| }, |
| { |
| "name": "model.layers.2.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 26429168 |
| }, |
| { |
| "name": "model.layers.2.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 26431728 |
| }, |
| { |
| "name": "model.layers.2.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 31019248 |
| } |
| ], |
| "md5sum": "71e2bc5e3ef0f7f5da5692f689d2d30c" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 32735488, |
| "records": [ |
| { |
| "name": "model.layers.2.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.2.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 2293760 |
| }, |
| { |
| "name": "model.layers.3.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 2580480 |
| }, |
| { |
| "name": "model.layers.3.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 2583040 |
| }, |
| { |
| "name": "model.layers.3.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 3811840 |
| }, |
| { |
| "name": "model.layers.3.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 3965440 |
| }, |
| { |
| "name": "model.layers.3.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 3965568 |
| }, |
| { |
| "name": "model.layers.3.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 3965696 |
| }, |
| { |
| "name": "model.layers.3.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 4784896 |
| }, |
| { |
| "name": "model.layers.3.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 4887296 |
| }, |
| { |
| "name": "model.layers.3.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 4889856 |
| }, |
| { |
| "name": "model.layers.3.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 9477376 |
| }, |
| { |
| "name": "model.layers.3.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.3.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 12344576 |
| }, |
| { |
| "name": "model.layers.4.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 12631296 |
| }, |
| { |
| "name": "model.layers.4.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 12633856 |
| }, |
| { |
| "name": "model.layers.4.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 13862656 |
| }, |
| { |
| "name": "model.layers.4.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 14016256 |
| }, |
| { |
| "name": "model.layers.4.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 14016384 |
| }, |
| { |
| "name": "model.layers.4.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 14016512 |
| }, |
| { |
| "name": "model.layers.4.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 14835712 |
| }, |
| { |
| "name": "model.layers.4.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 14938112 |
| }, |
| { |
| "name": "model.layers.4.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 14940672 |
| }, |
| { |
| "name": "model.layers.4.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 19528192 |
| }, |
| { |
| "name": "model.layers.4.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.4.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 22395392 |
| }, |
| { |
| "name": "model.layers.5.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 22682112 |
| }, |
| { |
| "name": "model.layers.5.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 22684672 |
| }, |
| { |
| "name": "model.layers.5.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 23913472 |
| }, |
| { |
| "name": "model.layers.5.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 24067072 |
| }, |
| { |
| "name": "model.layers.5.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 24067200 |
| }, |
| { |
| "name": "model.layers.5.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 24067328 |
| }, |
| { |
| "name": "model.layers.5.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 24886528 |
| }, |
| { |
| "name": "model.layers.5.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 24988928 |
| }, |
| { |
| "name": "model.layers.5.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 24991488 |
| }, |
| { |
| "name": "model.layers.5.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 29579008 |
| }, |
| { |
| "name": "model.layers.5.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 30152448 |
| }, |
| { |
| "name": "model.layers.5.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 32446208 |
| }, |
| { |
| "name": "model.layers.6.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 32732928 |
| } |
| ], |
| "md5sum": "f570f37f4a198537d8c8d5c4ea16f9d4" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 32459264, |
| "records": [ |
| { |
| "name": "model.layers.6.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.6.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 1228800 |
| }, |
| { |
| "name": "model.layers.6.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 1382400 |
| }, |
| { |
| "name": "model.layers.6.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 1382528 |
| }, |
| { |
| "name": "model.layers.6.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 1382656 |
| }, |
| { |
| "name": "model.layers.6.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 2201856 |
| }, |
| { |
| "name": "model.layers.6.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 2304256 |
| }, |
| { |
| "name": "model.layers.6.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 2306816 |
| }, |
| { |
| "name": "model.layers.6.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 6894336 |
| }, |
| { |
| "name": "model.layers.6.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 7467776 |
| }, |
| { |
| "name": "model.layers.6.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 9761536 |
| }, |
| { |
| "name": "model.layers.7.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.7.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.7.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 11279616 |
| }, |
| { |
| "name": "model.layers.7.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 11433216 |
| }, |
| { |
| "name": "model.layers.7.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 11433344 |
| }, |
| { |
| "name": "model.layers.7.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 11433472 |
| }, |
| { |
| "name": "model.layers.7.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 12252672 |
| }, |
| { |
| "name": "model.layers.7.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 12355072 |
| }, |
| { |
| "name": "model.layers.7.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 12357632 |
| }, |
| { |
| "name": "model.layers.7.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 16945152 |
| }, |
| { |
| "name": "model.layers.7.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 17518592 |
| }, |
| { |
| "name": "model.layers.7.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 19812352 |
| }, |
| { |
| "name": "model.layers.8.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.8.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.8.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 21330432 |
| }, |
| { |
| "name": "model.layers.8.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 21484032 |
| }, |
| { |
| "name": "model.layers.8.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 21484160 |
| }, |
| { |
| "name": "model.layers.8.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 21484288 |
| }, |
| { |
| "name": "model.layers.8.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 22303488 |
| }, |
| { |
| "name": "model.layers.8.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 22405888 |
| }, |
| { |
| "name": "model.layers.8.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 22408448 |
| }, |
| { |
| "name": "model.layers.8.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 26995968 |
| }, |
| { |
| "name": "model.layers.8.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 27569408 |
| }, |
| { |
| "name": "model.layers.8.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 29863168 |
| }, |
| { |
| "name": "model.layers.9.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| }, |
| { |
| "name": "model.layers.9.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 30152448 |
| }, |
| { |
| "name": "model.layers.9.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 31381248 |
| }, |
| { |
| "name": "model.layers.9.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 31534848 |
| }, |
| { |
| "name": "model.layers.9.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 31534976 |
| }, |
| { |
| "name": "model.layers.9.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 31535104 |
| }, |
| { |
| "name": "model.layers.9.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 32354304 |
| }, |
| { |
| "name": "model.layers.9.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 32456704 |
| } |
| ], |
| "md5sum": "ea30c26e10a90a76d049476c427d750c" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.9.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.9.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.9.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.9.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.10.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.10.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.10.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.10.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.10.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.10.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.10.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.10.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.10.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.10.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.10.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.10.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.11.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.11.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.11.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.11.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.11.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.11.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.11.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.11.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.11.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.11.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.11.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.11.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.12.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.12.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.12.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.12.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.12.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.12.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.12.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.12.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "cc5002560b2d46a4bf44999f185588e4" |
| }, |
| { |
| "dataPath": "params_shard_5.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.12.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.12.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.12.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.12.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.13.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.13.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.13.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.13.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.13.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.13.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.13.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.13.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.13.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.13.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.13.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.13.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.14.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.14.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.14.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.14.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.14.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.14.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.14.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.14.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.14.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.14.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.14.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.14.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.15.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.15.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.15.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.15.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.15.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.15.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.15.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.15.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "2a2e5f06d382e5f7d493d5103ca447ff" |
| }, |
| { |
| "dataPath": "params_shard_6.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.15.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.15.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.15.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.15.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.16.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.16.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.16.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.16.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.16.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.16.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.16.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.16.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.16.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.16.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.16.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.16.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.17.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.17.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.17.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.17.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.17.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.17.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.17.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.17.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.17.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.17.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.17.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.17.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.18.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.18.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.18.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.18.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.18.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.18.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.18.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.18.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "15f16f3bed054807624d3276b45a9bfb" |
| }, |
| { |
| "dataPath": "params_shard_7.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.18.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.18.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.18.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.18.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.19.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.19.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.19.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.19.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.19.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.19.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.19.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.19.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.19.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.19.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.19.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.19.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.20.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.20.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.20.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.20.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.20.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.20.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.20.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.20.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.20.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.20.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.20.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.20.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.21.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.21.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.21.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.21.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.21.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.21.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.21.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.21.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "b7fa2d8815a49dfe4ef62895f6656408" |
| }, |
| { |
| "dataPath": "params_shard_8.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.21.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.21.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.21.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.21.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.22.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.22.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.22.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.22.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.22.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.22.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.22.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.22.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.22.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.22.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.22.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.22.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.23.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.23.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.23.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.23.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.23.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.23.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.23.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.23.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.23.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.23.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.23.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.23.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.24.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.24.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.24.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.24.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.24.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.24.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.24.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.24.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "2c776a967ed91c03ef498beadb76eff3" |
| }, |
| { |
| "dataPath": "params_shard_9.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.24.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.24.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.24.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.24.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.25.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.25.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.25.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.25.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.25.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.25.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.25.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.25.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.25.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.25.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.25.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.25.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.26.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.26.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.26.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.26.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.26.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.26.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.26.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.26.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.26.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.26.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.26.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.26.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.27.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.27.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.27.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.27.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.27.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.27.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.27.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.27.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "e0c7246b8ddc51911b52b1811f55f1a2" |
| }, |
| { |
| "dataPath": "params_shard_10.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.27.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.27.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.27.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.27.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.28.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.28.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.28.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.28.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.28.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.28.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.28.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.28.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.28.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.28.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.28.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.28.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.29.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.29.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.29.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.29.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.29.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.29.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.29.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.29.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.29.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.29.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.29.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.29.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.30.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.30.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.30.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.30.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.30.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.30.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.30.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.30.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "bee32993f0b20df1f5d48d18fe3fe965" |
| }, |
| { |
| "dataPath": "params_shard_11.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.30.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.30.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.30.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.30.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.31.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.31.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.31.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.31.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.31.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.31.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.31.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.31.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.31.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.31.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.31.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.31.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.32.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.32.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.32.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.32.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.32.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.32.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.32.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.32.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.32.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.32.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.32.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.32.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.33.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.33.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.33.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.33.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.33.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.33.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.33.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.33.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "b45f138844d0829df7fc55d4cc9ad46a" |
| }, |
| { |
| "dataPath": "params_shard_12.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.33.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.33.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.33.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.33.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.34.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.34.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.34.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.34.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.34.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.34.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.34.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.34.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.34.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.34.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.34.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.34.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.35.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.35.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.35.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.35.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.35.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.35.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.35.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.35.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.35.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.35.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.35.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.35.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.36.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.36.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.36.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.36.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.36.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.36.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.36.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.36.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "3e2459b4930627759a5bb55effe1882a" |
| }, |
| { |
| "dataPath": "params_shard_13.bin", |
| "format": "raw-shard", |
| "nbytes": 30152448, |
| "records": [ |
| { |
| "name": "model.layers.36.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.36.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.36.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.36.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.layers.37.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| }, |
| { |
| "name": "model.layers.37.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 7744000 |
| }, |
| { |
| "name": "model.layers.37.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 8972800 |
| }, |
| { |
| "name": "model.layers.37.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126400 |
| }, |
| { |
| "name": "model.layers.37.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 9126528 |
| }, |
| { |
| "name": "model.layers.37.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 9126656 |
| }, |
| { |
| "name": "model.layers.37.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 9945856 |
| }, |
| { |
| "name": "model.layers.37.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 10048256 |
| }, |
| { |
| "name": "model.layers.37.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 10050816 |
| }, |
| { |
| "name": "model.layers.37.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 14638336 |
| }, |
| { |
| "name": "model.layers.37.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 15211776 |
| }, |
| { |
| "name": "model.layers.37.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 17505536 |
| }, |
| { |
| "name": "model.layers.38.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 17792256 |
| }, |
| { |
| "name": "model.layers.38.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 17794816 |
| }, |
| { |
| "name": "model.layers.38.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 19023616 |
| }, |
| { |
| "name": "model.layers.38.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177216 |
| }, |
| { |
| "name": "model.layers.38.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 19177344 |
| }, |
| { |
| "name": "model.layers.38.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 19177472 |
| }, |
| { |
| "name": "model.layers.38.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 19996672 |
| }, |
| { |
| "name": "model.layers.38.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 20099072 |
| }, |
| { |
| "name": "model.layers.38.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 20101632 |
| }, |
| { |
| "name": "model.layers.38.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 24689152 |
| }, |
| { |
| "name": "model.layers.38.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 25262592 |
| }, |
| { |
| "name": "model.layers.38.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 27556352 |
| }, |
| { |
| "name": "model.layers.39.input_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 27843072 |
| }, |
| { |
| "name": "model.layers.39.self_attn.c_attn.q_weight", |
| "shape": [ |
| 1920, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 1228800, |
| "byteOffset": 27845632 |
| }, |
| { |
| "name": "model.layers.39.self_attn.c_attn.q_scale", |
| "shape": [ |
| 1920, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 153600, |
| "byteOffset": 29074432 |
| }, |
| { |
| "name": "model.layers.39.self_attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228032 |
| }, |
| { |
| "name": "model.layers.39.self_attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 128, |
| "byteOffset": 29228160 |
| }, |
| { |
| "name": "model.layers.39.self_attn.o_proj.q_weight", |
| "shape": [ |
| 1280, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 819200, |
| "byteOffset": 29228288 |
| }, |
| { |
| "name": "model.layers.39.self_attn.o_proj.q_scale", |
| "shape": [ |
| 1280, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 102400, |
| "byteOffset": 30047488 |
| }, |
| { |
| "name": "model.layers.39.post_attention_layernorm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 30149888 |
| } |
| ], |
| "md5sum": "64b91dfda4fcb122ce15fa8d7358a7ec" |
| }, |
| { |
| "dataPath": "params_shard_14.bin", |
| "format": "raw-shard", |
| "nbytes": 7744000, |
| "records": [ |
| { |
| "name": "model.layers.39.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 7168, |
| 160 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4587520, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.39.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 7168, |
| 40 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 573440, |
| "byteOffset": 4587520 |
| }, |
| { |
| "name": "model.layers.39.mlp.down_proj.q_weight", |
| "shape": [ |
| 1280, |
| 448 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 2293760, |
| "byteOffset": 5160960 |
| }, |
| { |
| "name": "model.layers.39.mlp.down_proj.q_scale", |
| "shape": [ |
| 1280, |
| 112 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 286720, |
| "byteOffset": 7454720 |
| }, |
| { |
| "name": "model.norm.weight", |
| "shape": [ |
| 1280 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 2560, |
| "byteOffset": 7741440 |
| } |
| ], |
| "md5sum": "6d0f0fd803d2792e90b8e0489a5a77f3" |
| } |
| ] |
| } |
