| { |
| "metadata": { |
| "ParamSize": 437, |
| "ParamBytes": 2481708032.0, |
| "BitsPerParam": 4.500511196354994 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 194478080, |
| "records": [ |
| { |
| "name": "lm_head.q_weight", |
| "shape": [ |
| 151936, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 194478080, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "db2b8cb0213727790cdb0e5e0831d3da" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 24314880, |
| "records": [ |
| { |
| "name": "lm_head.q_scale", |
| "shape": [ |
| 151936, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 24309760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.20.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 24309760 |
| } |
| ], |
| "md5sum": "8e09688cd766658d34683deaa3bdaf15" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.21.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "ebafb35e880da1dd1a86bb28d0eaa634" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 31145216, |
| "records": [ |
| { |
| "name": "model.layers.20.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.20.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.20.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.21.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14013440 |
| }, |
| { |
| "name": "model.layers.21.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14018560 |
| }, |
| { |
| "name": "model.layers.21.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 26470400 |
| }, |
| { |
| "name": "model.layers.21.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28026880 |
| }, |
| { |
| "name": "model.layers.21.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31139840 |
| }, |
| { |
| "name": "model.layers.21.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31144960 |
| } |
| ], |
| "md5sum": "a474a9869b74df0b792c7cfecd1005a5" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.22.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "d7b4e73816f305d3f9db3a01d235ab72" |
| }, |
| { |
| "dataPath": "params_shard_5.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.21.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.21.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.21.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.21.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.21.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.22.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.22.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.22.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.22.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.22.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.22.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "277aaa96083cc98d47dda1490e1a6919" |
| }, |
| { |
| "dataPath": "params_shard_6.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.23.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "91ec6f1ec8233d7fb4215f398d7a4211" |
| }, |
| { |
| "dataPath": "params_shard_7.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.22.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.22.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.22.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.22.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.22.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.23.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.23.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.23.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.23.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.23.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.23.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "7b83d2c9d0b0c35f76c0debe056620b2" |
| }, |
| { |
| "dataPath": "params_shard_8.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.24.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "c0a5b2d46655cd1909514e144b02207c" |
| }, |
| { |
| "dataPath": "params_shard_9.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.23.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.23.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.23.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.23.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.23.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.24.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.24.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.24.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.24.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.24.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.24.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "ec993e1986197980a7032c79890c901c" |
| }, |
| { |
| "dataPath": "params_shard_10.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.25.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "4d87636ad72aadb367b2e36191f4e4d7" |
| }, |
| { |
| "dataPath": "params_shard_11.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.24.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.24.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.24.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.24.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.24.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.25.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.25.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.25.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.25.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.25.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.25.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "5485e9c6bb6e1b3b0a23c2f42c8fce19" |
| }, |
| { |
| "dataPath": "params_shard_12.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.26.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "afdb9619569950a49097607fb00f745c" |
| }, |
| { |
| "dataPath": "params_shard_13.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.25.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.25.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.25.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.25.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.25.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.26.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.26.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.26.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.26.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.26.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.26.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "a2e1be5be9d643c054c0b7162d340478" |
| }, |
| { |
| "dataPath": "params_shard_14.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.27.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "63713ebf0922eaf8a6d11b06ec1468f7" |
| }, |
| { |
| "dataPath": "params_shard_15.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.26.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.26.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.26.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.26.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.26.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.27.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.27.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.27.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.27.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.27.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.27.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "fb881f6775d260bc98505f8c89981a7d" |
| }, |
| { |
| "dataPath": "params_shard_16.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.28.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "7c21e07ce47ab803cddeb3814b3a5f83" |
| }, |
| { |
| "dataPath": "params_shard_17.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.27.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.27.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.27.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.27.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.27.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.28.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.28.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.28.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.28.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.28.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.28.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "544ca4403ae8275ce655968542605ca9" |
| }, |
| { |
| "dataPath": "params_shard_18.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.29.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "ca1bca7d66513e14c04f873417acfdaf" |
| }, |
| { |
| "dataPath": "params_shard_19.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.28.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.28.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.28.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.28.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.28.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.29.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.29.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.29.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.29.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.29.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.29.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "a698140b92be5f76cf0581fad7ce1a26" |
| }, |
| { |
| "dataPath": "params_shard_20.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.30.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e9927fc7d7e82288d5ab9bc8b5792e2d" |
| }, |
| { |
| "dataPath": "params_shard_21.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.29.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.29.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.29.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.29.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.29.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.30.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.30.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.30.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.30.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.30.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.30.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "074a0493d3e7db4325a15bd6e2fe9ba6" |
| }, |
| { |
| "dataPath": "params_shard_22.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.31.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "9b83e79558c86d49c9606c66d9ea258d" |
| }, |
| { |
| "dataPath": "params_shard_23.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.30.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.30.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.30.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.30.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.30.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.31.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.31.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.31.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.31.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.31.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.31.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "7b35cfea6c34831e72a11373c152ef6f" |
| }, |
| { |
| "dataPath": "params_shard_24.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.32.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "948d6b6cda2c6c36ee8069a53f5e899d" |
| }, |
| { |
| "dataPath": "params_shard_25.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.31.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.31.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.31.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.31.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.31.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.32.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.32.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.32.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.32.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.32.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.32.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "e5acfd39577eb2586dab938608f0b5fe" |
| }, |
| { |
| "dataPath": "params_shard_26.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.33.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "c84944b2402b0351218891a54d1368ce" |
| }, |
| { |
| "dataPath": "params_shard_27.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.32.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.32.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.32.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.32.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.32.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.33.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.33.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.33.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.33.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.33.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.33.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "a98f9ebcbef9c0ac9b7d9ed5be3a7f6d" |
| }, |
| { |
| "dataPath": "params_shard_28.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.34.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "d9bda2ac4626a6b61fd7d2d2761daf3b" |
| }, |
| { |
| "dataPath": "params_shard_29.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.33.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.33.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.33.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.33.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.33.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.34.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.34.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.34.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.34.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.34.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.34.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "53208dff9e2b730b60fa6b538161d3bd" |
| }, |
| { |
| "dataPath": "params_shard_30.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.35.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "25af209c1cc13ec0d87923e976ac6f78" |
| }, |
| { |
| "dataPath": "params_shard_31.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.34.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.34.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.34.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.34.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.34.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.35.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.35.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.35.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.35.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.35.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.35.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "da31b35be007bc543e09983f62baa37a" |
| }, |
| { |
| "dataPath": "params_shard_32.bin", |
| "format": "raw-shard", |
| "nbytes": 194478080, |
| "records": [ |
| { |
| "name": "model.embed_tokens.q_weight", |
| "shape": [ |
| 151936, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 194478080, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "82277c94f44a3a7153639891038ecfb0" |
| }, |
| { |
| "dataPath": "params_shard_33.bin", |
| "format": "raw-shard", |
| "nbytes": 24309760, |
| "records": [ |
| { |
| "name": "model.embed_tokens.q_scale", |
| "shape": [ |
| 151936, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 24309760, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "7c3bc9c7d7bcdd2aa47432f556a99dc5" |
| }, |
| { |
| "dataPath": "params_shard_34.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.0.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "4a7e127455416e86f350fa1f0c2d8e72" |
| }, |
| { |
| "dataPath": "params_shard_35.bin", |
| "format": "raw-shard", |
| "nbytes": 31882752, |
| "records": [ |
| { |
| "name": "model.layers.35.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.35.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.35.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.35.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.35.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.norm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.0.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.0.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14756096 |
| }, |
| { |
| "name": "model.layers.0.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27207936 |
| }, |
| { |
| "name": "model.layers.0.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28764416 |
| }, |
| { |
| "name": "model.layers.0.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31877376 |
| }, |
| { |
| "name": "model.layers.0.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31882496 |
| } |
| ], |
| "md5sum": "e192ee861fe1e3907dceb431947e4350" |
| }, |
| { |
| "dataPath": "params_shard_36.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.1.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "99f64762f8e041a8e35eccb933ab7768" |
| }, |
| { |
| "dataPath": "params_shard_37.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.0.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.0.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.0.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.0.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.0.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.1.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.1.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.1.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.1.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.1.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.1.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "11bf858f5d65582ea6d190cd7e7dade8" |
| }, |
| { |
| "dataPath": "params_shard_38.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.10.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "7f1d26463cdfa648afcedb11110aea1c" |
| }, |
| { |
| "dataPath": "params_shard_39.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.1.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.1.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.1.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.1.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.1.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.10.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.10.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.10.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.10.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.10.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.10.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "d110d0706b7420b0dce14651950800a7" |
| }, |
| { |
| "dataPath": "params_shard_40.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.11.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "c7f1aa652746c1e5b43fcd791c4229db" |
| }, |
| { |
| "dataPath": "params_shard_41.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.10.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.10.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.10.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.10.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.10.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.11.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.11.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.11.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.11.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.11.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.11.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "094695dd5d449384b12ea5ba23cfbb85" |
| }, |
| { |
| "dataPath": "params_shard_42.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.12.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "edf74eef64f29760519e7513a8388e7c" |
| }, |
| { |
| "dataPath": "params_shard_43.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.11.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.11.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.11.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.11.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.11.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.12.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.12.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.12.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.12.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.12.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.12.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "a19d26bd7b650955692a794f529fc513" |
| }, |
| { |
| "dataPath": "params_shard_44.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.13.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "49fe9f1430285032069c36c30797c4da" |
| }, |
| { |
| "dataPath": "params_shard_45.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.12.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.12.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.12.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.12.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.12.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.13.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.13.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.13.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.13.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.13.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.13.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "665d2f55de5ce85f19bc447d3b5d69b9" |
| }, |
| { |
| "dataPath": "params_shard_46.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.14.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "efdd6eda73a5841267ee33595e1d42c3" |
| }, |
| { |
| "dataPath": "params_shard_47.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.13.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.13.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.13.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.13.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.13.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.14.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.14.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.14.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.14.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.14.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.14.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "acbda2d22c29dfa074168e5078dda0fc" |
| }, |
| { |
| "dataPath": "params_shard_48.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.15.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "20f827e04f653d2b2b4f223c5b99c227" |
| }, |
| { |
| "dataPath": "params_shard_49.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.14.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.14.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.14.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.14.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.14.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.15.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.15.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.15.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.15.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.15.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.15.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "03d70f5d2827e84699c96e5245e33d08" |
| }, |
| { |
| "dataPath": "params_shard_50.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.16.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "1cbf527260733263d976a15d77bae615" |
| }, |
| { |
| "dataPath": "params_shard_51.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.15.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.15.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.15.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.15.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.15.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.16.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.16.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.16.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.16.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.16.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.16.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "a04ae63eace20a1fa6bc27485d3f83e9" |
| }, |
| { |
| "dataPath": "params_shard_52.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.17.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "3dba22d3bba03d66d0c20bcbb9719275" |
| }, |
| { |
| "dataPath": "params_shard_53.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.16.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.16.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.16.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.16.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.16.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.17.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.17.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.17.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.17.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.17.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.17.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "5efe09c1541eebc649f233c6a6b2da83" |
| }, |
| { |
| "dataPath": "params_shard_54.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.18.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "0cc0fd5454bfa4ece259c5c7c5b0c8c5" |
| }, |
| { |
| "dataPath": "params_shard_55.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.17.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.17.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.17.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.17.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.17.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.18.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.18.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.18.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.18.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.18.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.18.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "614af42de5c906d965c52c47ec4c6094" |
| }, |
| { |
| "dataPath": "params_shard_56.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.19.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "ea4f07a62c5d0ececb7819ab2e07b35d" |
| }, |
| { |
| "dataPath": "params_shard_57.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.18.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.18.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.18.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.18.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.18.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.19.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.19.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.19.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.19.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.19.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.19.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "8a0a787ec787f1b270ee33b07760df90" |
| }, |
| { |
| "dataPath": "params_shard_58.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.2.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f76c373fb865ef58db8d0a6fb03bb047" |
| }, |
| { |
| "dataPath": "params_shard_59.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.19.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.19.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.19.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.19.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.19.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.2.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.2.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 14750976 |
| }, |
| { |
| "name": "model.layers.2.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 27202816 |
| }, |
| { |
| "name": "model.layers.2.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 28759296 |
| }, |
| { |
| "name": "model.layers.2.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.2.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31877376 |
| } |
| ], |
| "md5sum": "53fd39b7db41f584018f4be8ce993dd4" |
| }, |
| { |
| "dataPath": "params_shard_60.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.20.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "cd29c01841e42daf299feb5646ae8e3e" |
| }, |
| { |
| "dataPath": "params_shard_61.bin", |
| "format": "raw-shard", |
| "nbytes": 32610048, |
| "records": [ |
| { |
| "name": "model.layers.2.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.2.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 7864320 |
| }, |
| { |
| "name": "model.layers.2.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 8847360 |
| }, |
| { |
| "name": "model.layers.2.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 14090240 |
| }, |
| { |
| "name": "model.layers.2.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 14745600 |
| }, |
| { |
| "name": "model.layers.20.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14745856 |
| }, |
| { |
| "name": "model.layers.20.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17858816 |
| }, |
| { |
| "name": "model.layers.20.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17859072 |
| }, |
| { |
| "name": "model.layers.20.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 25723392 |
| }, |
| { |
| "name": "model.layers.20.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 26706432 |
| }, |
| { |
| "name": "model.layers.20.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31949312 |
| }, |
| { |
| "name": "model.layers.20.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 32604672 |
| }, |
| { |
| "name": "model.layers.3.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 32604928 |
| } |
| ], |
| "md5sum": "44ac92fff9d9700bb386de52cd784d02" |
| }, |
| { |
| "dataPath": "params_shard_62.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.3.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "6079939593d71d90af338c0bc65f9081" |
| }, |
| { |
| "dataPath": "params_shard_63.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.3.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.3.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.3.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.3.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.3.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.3.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.3.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.3.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.3.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.3.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.4.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "cfc26bc06631245b44a9c6692f0e7e35" |
| }, |
| { |
| "dataPath": "params_shard_64.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.4.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "444704eb41944633c388985da1054e21" |
| }, |
| { |
| "dataPath": "params_shard_65.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.4.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.4.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.4.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.4.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.4.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.4.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.4.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.4.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.4.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.4.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.5.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "5425a49226308f79f16a767dcda55ae1" |
| }, |
| { |
| "dataPath": "params_shard_66.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.5.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "a347e6c9c633653b28dde5c2ff5677e9" |
| }, |
| { |
| "dataPath": "params_shard_67.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.5.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.5.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.5.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.5.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.5.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.5.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.5.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.5.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.5.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.5.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.6.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "a35de5c6c6f88554f44bbeddb6371f76" |
| }, |
| { |
| "dataPath": "params_shard_68.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.6.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "c82410e1decfe08ee014bccb4283b1eb" |
| }, |
| { |
| "dataPath": "params_shard_69.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.6.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.6.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.6.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.6.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.6.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.6.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.6.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.6.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.6.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.6.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.7.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "72d3864f5111d48f1a07272cb24ca756" |
| }, |
| { |
| "dataPath": "params_shard_70.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.7.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "1ded7b18d6c5ab237cbbacea92ca1532" |
| }, |
| { |
| "dataPath": "params_shard_71.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.7.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.7.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.7.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.7.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.7.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.7.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.7.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.7.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.7.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.7.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.8.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "32b6b99a610c1837c85e7d7c0c494cfd" |
| }, |
| { |
| "dataPath": "params_shard_72.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.8.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "2a10028adfc046ff918b68eca2eebcd6" |
| }, |
| { |
| "dataPath": "params_shard_73.bin", |
| "format": "raw-shard", |
| "nbytes": 31877632, |
| "records": [ |
| { |
| "name": "model.layers.8.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.8.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.8.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.8.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.8.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.8.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.8.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.8.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.8.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.8.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| }, |
| { |
| "name": "model.layers.9.input_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 31872512 |
| } |
| ], |
| "md5sum": "b5078d97ecf28a3ce7e31e836de46141" |
| }, |
| { |
| "dataPath": "params_shard_74.bin", |
| "format": "raw-shard", |
| "nbytes": 24903680, |
| "records": [ |
| { |
| "name": "model.layers.9.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 19456, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 24903680, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "c9dbfda92afd0cd7c350572901579d88" |
| }, |
| { |
| "dataPath": "params_shard_75.bin", |
| "format": "raw-shard", |
| "nbytes": 31872512, |
| "records": [ |
| { |
| "name": "model.layers.9.mlp.down_proj.q_weight", |
| "shape": [ |
| 2560, |
| 1216 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12451840, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "model.layers.9.mlp.down_proj.q_scale", |
| "shape": [ |
| 2560, |
| 304 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1556480, |
| "byteOffset": 12451840 |
| }, |
| { |
| "name": "model.layers.9.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 19456, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3112960, |
| "byteOffset": 14008320 |
| }, |
| { |
| "name": "model.layers.9.post_attention_layernorm.weight", |
| "shape": [ |
| 2560 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 5120, |
| "byteOffset": 17121280 |
| }, |
| { |
| "name": "model.layers.9.self_attn.k_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 17126400 |
| }, |
| { |
| "name": "model.layers.9.self_attn.c_attn.q_weight", |
| "shape": [ |
| 6144, |
| 320 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 7864320, |
| "byteOffset": 17126656 |
| }, |
| { |
| "name": "model.layers.9.self_attn.c_attn.q_scale", |
| "shape": [ |
| 6144, |
| 80 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 983040, |
| "byteOffset": 24990976 |
| }, |
| { |
| "name": "model.layers.9.self_attn.o_proj.q_weight", |
| "shape": [ |
| 2560, |
| 512 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 5242880, |
| "byteOffset": 25974016 |
| }, |
| { |
| "name": "model.layers.9.self_attn.o_proj.q_scale", |
| "shape": [ |
| 2560, |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 655360, |
| "byteOffset": 31216896 |
| }, |
| { |
| "name": "model.layers.9.self_attn.q_norm.weight", |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 256, |
| "byteOffset": 31872256 |
| } |
| ], |
| "md5sum": "dfb4f84202294de5b821ea7499e31e2c" |
| } |
| ] |
| } |
