{ "metadata": { "ParamSize": 76, "ParamBytes": 140883968.0, "BitsPerParam": 16.00348958939165 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "gpt_neox.embed_in.weight", "shape": [ 50304, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "a1619d49d0b0ba376d0f671d12c13e92" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 33103872, "records": [ { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 1024 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 2048 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 3072 }, { "name": "gpt_neox.layers.0.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 4096 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 1576960 }, { "name": "gpt_neox.layers.0.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 1580032 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 2104320 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2105344 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4202496 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 4206592 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 6303744 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 6304768 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 6305792 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 6306816 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 6307840 }, { "name": "gpt_neox.layers.1.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 6308864 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 7881728 }, { "name": "gpt_neox.layers.1.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 7884800 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 8409088 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 8410112 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 10507264 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 10511360 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 12608512 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 12609536 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 12610560 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 12611584 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 12612608 }, { "name": "gpt_neox.layers.2.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 12613632 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 14186496 }, { "name": "gpt_neox.layers.2.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 14189568 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 14713856 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 14714880 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 16812032 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16816128 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18913280 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18914304 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18915328 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18916352 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18917376 }, { "name": "gpt_neox.layers.3.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 18918400 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 20491264 }, { "name": "gpt_neox.layers.3.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 20494336 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21018624 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21019648 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 23116800 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23120896 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25218048 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25219072 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25220096 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25221120 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25222144 }, { "name": "gpt_neox.layers.4.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 25223168 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 26796032 }, { "name": "gpt_neox.layers.4.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 26799104 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 27323392 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27324416 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 29421568 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 29425664 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 31522816 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 31523840 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 31524864 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 31525888 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 31526912 }, { "name": "gpt_neox.layers.5.attention.query_key_value.weight", "shape": [ 1536, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 31527936 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 33100800 } ], "md5sum": "ef146e3d692ff0f92b4887051f5ce1aa" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "embed_out.weight", "shape": [ 50304, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "2f016b975538d1e27f57fc84596c2432" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 4726784, "records": [ { "name": "gpt_neox.layers.5.attention.dense.weight", "shape": [ 512, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 524288 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 525312 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 2622464 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", "shape": [ 512, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2626560 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 4723712 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 4724736 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 4725760 } ], "md5sum": "4b204eca2568a39d4f462b042c0e9ab7" } ] }