| { |
| "metadata": { |
| "ParamSize": 218, |
| "ParamBytes": 1072392704.0, |
| "BitsPerParam": 32.0 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 671088640, |
| "records": [ |
| { |
| "name": "language_model.model.embed_tokens.weight", |
| "shape": [ |
| 262144, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 671088640, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "dc61b2dbde6b034d5b7726300ba2bbe6" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 27539968, |
| "records": [ |
| { |
| "name": "language_model.model.layers.0.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 2560 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 5245440 |
| }, |
| { |
| "name": "language_model.model.layers.0.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 15731200 |
| }, |
| { |
| "name": "language_model.model.layers.0.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 15733760 |
| }, |
| { |
| "name": "language_model.model.layers.0.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 15736320 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 15738880 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 15739904 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 16395264 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 19016704 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 19017728 |
| }, |
| { |
| "name": "language_model.model.layers.0.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.1.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 22297088 |
| } |
| ], |
| "md5sum": "db92957e5f38ac46b880e648819040dd" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.1.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.1.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.1.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.1.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.1.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.10.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.10.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.10.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.10.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.10.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.10.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "f0cdda6253df27411398b7f2ca55c20d" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 33439232, |
| "records": [ |
| { |
| "name": "language_model.model.layers.10.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.10.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.10.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.10.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.layers.11.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 5901824 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 11144704 |
| }, |
| { |
| "name": "language_model.model.layers.11.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21630464 |
| }, |
| { |
| "name": "language_model.model.layers.11.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21633024 |
| }, |
| { |
| "name": "language_model.model.layers.11.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21635584 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 21638144 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 24915968 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 24916992 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 27538432 |
| }, |
| { |
| "name": "language_model.model.layers.12.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 28193792 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 28196352 |
| } |
| ], |
| "md5sum": "67d8105d19337ff42c9f7a9e9adf70e7" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.12.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.12.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.12.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.12.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.12.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.13.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.13.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.13.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.13.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.13.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.13.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "9f91b5b1350459df2806657d22f0a095" |
| }, |
| { |
| "dataPath": "params_shard_5.bin", |
| "format": "raw-shard", |
| "nbytes": 33439232, |
| "records": [ |
| { |
| "name": "language_model.model.layers.13.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.13.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.13.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.13.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.layers.14.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 5901824 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 11144704 |
| }, |
| { |
| "name": "language_model.model.layers.14.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21630464 |
| }, |
| { |
| "name": "language_model.model.layers.14.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21633024 |
| }, |
| { |
| "name": "language_model.model.layers.14.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21635584 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 21638144 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 24915968 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 24916992 |
| }, |
| { |
| "name": "language_model.model.layers.14.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 27538432 |
| }, |
| { |
| "name": "language_model.model.layers.15.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 28193792 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 28196352 |
| } |
| ], |
| "md5sum": "4d98f933fdbb71c2eaa1ec154cccfe69" |
| }, |
| { |
| "dataPath": "params_shard_6.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.15.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.15.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.15.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.15.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.16.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.16.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.16.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.16.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.16.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.16.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "a8c240e5a94b1661f5d04c233906c104" |
| }, |
| { |
| "dataPath": "params_shard_7.bin", |
| "format": "raw-shard", |
| "nbytes": 33439232, |
| "records": [ |
| { |
| "name": "language_model.model.layers.16.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.16.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.16.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.16.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.layers.17.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 5901824 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 11144704 |
| }, |
| { |
| "name": "language_model.model.layers.17.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21630464 |
| }, |
| { |
| "name": "language_model.model.layers.17.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21633024 |
| }, |
| { |
| "name": "language_model.model.layers.17.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21635584 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 21638144 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 24915968 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 24916992 |
| }, |
| { |
| "name": "language_model.model.layers.17.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 27538432 |
| }, |
| { |
| "name": "language_model.model.layers.2.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 28193792 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 28196352 |
| } |
| ], |
| "md5sum": "7c520ba0ed2bcd55481a4cc76a4ccc19" |
| }, |
| { |
| "dataPath": "params_shard_8.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.2.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.2.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.2.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.2.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.2.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.3.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.3.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.3.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.3.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "bc07cc11ff8fc6a08c61013350bc81b0" |
| }, |
| { |
| "dataPath": "params_shard_9.bin", |
| "format": "raw-shard", |
| "nbytes": 33439232, |
| "records": [ |
| { |
| "name": "language_model.model.layers.3.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.layers.4.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 5901824 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 11144704 |
| }, |
| { |
| "name": "language_model.model.layers.4.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21630464 |
| }, |
| { |
| "name": "language_model.model.layers.4.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21633024 |
| }, |
| { |
| "name": "language_model.model.layers.4.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21635584 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 21638144 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 24915968 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 24916992 |
| }, |
| { |
| "name": "language_model.model.layers.4.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 27538432 |
| }, |
| { |
| "name": "language_model.model.layers.5.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 28193792 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 28196352 |
| } |
| ], |
| "md5sum": "45c4d9bc7408e84a08965b347046617b" |
| }, |
| { |
| "dataPath": "params_shard_10.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.5.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.5.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.5.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.5.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.5.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.6.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.6.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.6.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.6.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.6.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.6.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "6615fb022c3490908b0f43d89fa441cf" |
| }, |
| { |
| "dataPath": "params_shard_11.bin", |
| "format": "raw-shard", |
| "nbytes": 33439232, |
| "records": [ |
| { |
| "name": "language_model.model.layers.6.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.6.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.6.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.6.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.layers.7.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 5901824 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 11144704 |
| }, |
| { |
| "name": "language_model.model.layers.7.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21630464 |
| }, |
| { |
| "name": "language_model.model.layers.7.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21633024 |
| }, |
| { |
| "name": "language_model.model.layers.7.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 21635584 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 21638144 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 21639168 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 24915968 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 24916992 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 27538432 |
| }, |
| { |
| "name": "language_model.model.layers.8.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 28193792 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 28196352 |
| } |
| ], |
| "md5sum": "e9b74bc9c8eafd7dd1ce4be73f0d68c5" |
| }, |
| { |
| "dataPath": "params_shard_12.bin", |
| "format": "raw-shard", |
| "nbytes": 33444352, |
| "records": [ |
| { |
| "name": "language_model.model.layers.8.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.8.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10485760 |
| }, |
| { |
| "name": "language_model.model.layers.8.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10488320 |
| }, |
| { |
| "name": "language_model.model.layers.8.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 10490880 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 10493440 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 10494464 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 11149824 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 13771264 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 13772288 |
| }, |
| { |
| "name": "language_model.model.layers.8.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 16393728 |
| }, |
| { |
| "name": "language_model.model.layers.9.input_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 17049088 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.down_proj.weight", |
| "shape": [ |
| 640, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 5242880, |
| "byteOffset": 17051648 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.gate_up_proj.weight", |
| "shape": [ |
| 4096, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 10485760, |
| "byteOffset": 22294528 |
| }, |
| { |
| "name": "language_model.model.layers.9.post_attention_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32780288 |
| }, |
| { |
| "name": "language_model.model.layers.9.post_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32782848 |
| }, |
| { |
| "name": "language_model.model.layers.9.pre_feedforward_layernorm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 32785408 |
| }, |
| { |
| "name": "language_model.model.layers.9.self_attn.k_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 32787968 |
| }, |
| { |
| "name": "language_model.model.layers.9.self_attn.k_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 32788992 |
| } |
| ], |
| "md5sum": "b50e8fd4079ebda7619cf73c0d544ccb" |
| }, |
| { |
| "dataPath": "params_shard_13.bin", |
| "format": "raw-shard", |
| "nbytes": 5901824, |
| "records": [ |
| { |
| "name": "language_model.model.layers.9.self_attn.o_proj.weight", |
| "shape": [ |
| 640, |
| 1024 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "language_model.model.layers.9.self_attn.q_norm.weight", |
| "shape": [ |
| 256 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1024, |
| "byteOffset": 2621440 |
| }, |
| { |
| "name": "language_model.model.layers.9.self_attn.q_proj.weight", |
| "shape": [ |
| 1024, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2621440, |
| "byteOffset": 2622464 |
| }, |
| { |
| "name": "language_model.model.layers.9.self_attn.v_proj.weight", |
| "shape": [ |
| 256, |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 655360, |
| "byteOffset": 5243904 |
| }, |
| { |
| "name": "language_model.model.norm.weight", |
| "shape": [ |
| 640 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2560, |
| "byteOffset": 5899264 |
| } |
| ], |
| "md5sum": "5a23d75cafcf542f60af43a71f00fa3f" |
| } |
| ] |
| } |
