pythia-160m / ndarray-cache.json
Amai
Add weights
9476ea2
Raw History Blame Contribute Delete
56.5 kB
{ "metadata": { "ParamSize": 148, "ParamBytes": 324738048.0, "BitsPerParam": 16.00454205660538 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 77266944, "records": [ { "name": "gpt_neox.embed_in.weight", "shape": [ 50304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 77266944, "byteOffset": 0 } ], "md5sum": "2481121002c12eed2951b729fa17b644" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 33082368, "records": [ { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1536 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 3072 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4608 }, { "name": "gpt_neox.layers.0.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 6144 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 3545088 }, { "name": "gpt_neox.layers.0.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 3549696 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4729344 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 4730880 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 9455616 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14174208 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14175744 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14177280 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14178816 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14180352 }, { "name": "gpt_neox.layers.1.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 14181888 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 17720832 }, { "name": "gpt_neox.layers.1.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 17725440 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18905088 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 18906624 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23625216 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 23631360 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28349952 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28351488 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28353024 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28354560 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28356096 }, { "name": "gpt_neox.layers.2.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 28357632 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 31896576 }, { "name": "gpt_neox.layers.2.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 31901184 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33080832 } ], "md5sum": "f152a892aaec5607fd2b89419435780a" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 33076224, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 4724736 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9443328 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9444864 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9446400 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9447936 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.3.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 9451008 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 12989952 }, { "name": "gpt_neox.layers.3.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 12994560 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14174208 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 14175744 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18894336 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 18900480 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23619072 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23620608 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23622144 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23623680 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23625216 }, { "name": "gpt_neox.layers.4.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 23626752 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 27165696 }, { "name": "gpt_neox.layers.4.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 27170304 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28349952 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 28351488 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33070080 } ], "md5sum": "57ece0af7da914417b0f9080c0bdac26" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 33077760, "records": [ { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4718592 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4720128 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4721664 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4723200 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4724736 }, { "name": "gpt_neox.layers.5.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 4726272 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 8265216 }, { "name": "gpt_neox.layers.5.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 8269824 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 9451008 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 14169600 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 14175744 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18894336 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18895872 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18897408 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18898944 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18900480 }, { "name": "gpt_neox.layers.6.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 18902016 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 22440960 }, { "name": "gpt_neox.layers.6.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 22445568 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23625216 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 23626752 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 28345344 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 28351488 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33070080 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33071616 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33073152 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33074688 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33076224 } ], "md5sum": "773204e0e5615a84b4109b95d21abb1e" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 33076224, "records": [ { "name": "gpt_neox.layers.7.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 3538944 }, { "name": "gpt_neox.layers.7.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 3543552 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4723200 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 4724736 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 9443328 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14168064 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14169600 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14171136 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14172672 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14174208 }, { "name": "gpt_neox.layers.8.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 14175744 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 17714688 }, { "name": "gpt_neox.layers.8.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 17719296 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18898944 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 18900480 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23619072 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 23625216 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28343808 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28345344 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28346880 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28348416 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28349952 }, { "name": "gpt_neox.layers.9.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 28351488 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 31890432 }, { "name": "gpt_neox.layers.9.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 31895040 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33074688 } ], "md5sum": "9be3a171e75708baaaf6249630cc7a0d" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 33076224, "records": [ { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4718592 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 4724736 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9443328 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9444864 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9446400 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9447936 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.10.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 9451008 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 12989952 }, { "name": "gpt_neox.layers.10.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 12994560 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14174208 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 14175744 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18894336 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 18900480 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23619072 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23620608 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23622144 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23623680 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23625216 }, { "name": "gpt_neox.layers.11.attention.query_key_value.weight", "shape": [ 2304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3538944, "byteOffset": 23626752 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 27165696 }, { "name": "gpt_neox.layers.11.attention.dense.weight", "shape": [ 768, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 27170304 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 28349952 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight", "shape": [ 3072, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 28351488 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 33070080 } ], "md5sum": "d3807c10c1e4cf1ed703d786dad97dd5" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 77266944, "records": [ { "name": "embed_out.weight", "shape": [ 50304, 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 77266944, "byteOffset": 0 } ], "md5sum": "084dbad7777340578a2501ea88f226dd" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 4723200, "records": [ { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight", "shape": [ 768, 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4718592, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4718592 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4720128 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 4721664 } ], "md5sum": "5290d766289dc0d43e7407c8dc728b2e" } ] }
{
"metadata": {
"ParamSize": 148,
"ParamBytes": 324738048.0,
"BitsPerParam": 16.00454205660538
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 77266944,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 77266944,
"byteOffset": 0
}
],
"md5sum": "2481121002c12eed2951b729fa17b644"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33082368,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1536
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 3072
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4608
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 6144
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 3545088
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",