{ "metadata": { "ParamSize": 390, "ParamBytes": 228708352.0, "BitsPerParam": 4.513973029098056 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 33447936, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50304, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25755648, "byteOffset": 0 }, { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50304, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3219456, "byteOffset": 25755648 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28975104 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28977152 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28979200 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28981248 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28983296 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 30556160 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 30752768 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 30758912 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 31283200 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31348736 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 31350784 } ], "md5sum": "4c3b2ea2f5c2a75292aa897ba7f70428" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 33425408, "records": [ { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 262144 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 270336 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 2367488 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2629632 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2631680 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2633728 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2635776 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2637824 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 2639872 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 4212736 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4409344 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 4415488 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 4939776 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 5005312 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 5007360 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 7104512 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 7366656 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7374848 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9472000 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9734144 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9736192 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9738240 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9740288 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9742336 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 9744384 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 11317248 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 11513856 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 11520000 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 12044288 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 12109824 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 12111872 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 14209024 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 14471168 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 14479360 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 16576512 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16838656 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16840704 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16842752 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16844800 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16846848 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 16848896 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 18421760 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18618368 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 18624512 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 19148800 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 19214336 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 19216384 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 21313536 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 21575680 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21583872 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 23681024 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23943168 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23945216 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23947264 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23949312 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23951360 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 23953408 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 25526272 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 25722880 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 25729024 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 26253312 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26318848 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26320896 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28418048 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 28680192 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28688384 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 30785536 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31047680 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31049728 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31051776 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31053824 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 31055872 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 31057920 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 32630784 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 32827392 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 32833536 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 33357824 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33423360 } ], "md5sum": "84fc60d24a69d34bf755e43043b00663" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 33155072, "records": [ { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 2097152 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 2359296 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2367488 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 4464640 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4726784 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4728832 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4730880 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4732928 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4734976 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 4737024 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 6309888 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6506496 }, { "name": "gpt_neox.layers.6.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 6512640 }, { "name": "gpt_neox.layers.6.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 7036928 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7102464 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7104512 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9201664 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 9463808 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 9472000 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11569152 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11831296 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11833344 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11835392 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11837440 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11839488 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 11841536 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 13414400 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 13611008 }, { "name": "gpt_neox.layers.7.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 13617152 }, { "name": "gpt_neox.layers.7.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 14141440 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14206976 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 14209024 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 16306176 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 16568320 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16576512 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18673664 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18935808 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18937856 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18939904 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18941952 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18944000 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 18946048 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 20518912 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 20715520 }, { "name": "gpt_neox.layers.8.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 20721664 }, { "name": "gpt_neox.layers.8.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 21245952 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21311488 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21313536 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 23410688 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 23672832 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23681024 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 25778176 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26040320 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26042368 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26044416 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26046464 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26048512 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 26050560 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 27623424 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 27820032 }, { "name": "gpt_neox.layers.9.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 27826176 }, { "name": "gpt_neox.layers.9.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 28350464 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28416000 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28418048 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 30515200 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 30777344 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 30785536 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 32882688 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33144832 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33146880 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33148928 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33150976 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33153024 } ], "md5sum": "685a0681b04bbc1cbba3e191ca3ba73b" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 33153024, "records": [ { "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 0 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 1572864 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 1769472 }, { "name": "gpt_neox.layers.10.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 1775616 }, { "name": "gpt_neox.layers.10.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 2299904 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2365440 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2367488 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 4464640 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 4726784 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 4734976 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 6832128 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7094272 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7096320 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7098368 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7100416 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7102464 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 7104512 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 8677376 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 8873984 }, { "name": "gpt_neox.layers.11.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 8880128 }, { "name": "gpt_neox.layers.11.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 9404416 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9469952 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 9472000 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11569152 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 11831296 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 11839488 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 13936640 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14198784 }, { "name": "gpt_neox.layers.12.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14200832 }, { "name": "gpt_neox.layers.12.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14202880 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14204928 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14206976 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 14209024 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 15781888 }, { "name": "gpt_neox.layers.12.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 15978496 }, { "name": "gpt_neox.layers.12.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 15984640 }, { "name": "gpt_neox.layers.12.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 16508928 }, { "name": "gpt_neox.layers.12.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16574464 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16576512 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18673664 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 18935808 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 18944000 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 21041152 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21303296 }, { "name": "gpt_neox.layers.13.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21305344 }, { "name": "gpt_neox.layers.13.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21307392 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21309440 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21311488 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 21313536 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 22886400 }, { "name": "gpt_neox.layers.13.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23083008 }, { "name": "gpt_neox.layers.13.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 23089152 }, { "name": "gpt_neox.layers.13.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 23613440 }, { "name": "gpt_neox.layers.13.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23678976 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23681024 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 25778176 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 26040320 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26048512 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28145664 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28407808 }, { "name": "gpt_neox.layers.14.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28409856 }, { "name": "gpt_neox.layers.14.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28411904 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28413952 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28416000 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 28418048 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 29990912 }, { "name": "gpt_neox.layers.14.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 30187520 }, { "name": "gpt_neox.layers.14.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 30193664 }, { "name": "gpt_neox.layers.14.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 30717952 }, { "name": "gpt_neox.layers.14.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30783488 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 30785536 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 32882688 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 33144832 } ], "md5sum": "a5fd0090ee7c8de95c415675151b3473" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 33155072, "records": [ { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 0 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 2097152 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2359296 }, { "name": "gpt_neox.layers.15.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2361344 }, { "name": "gpt_neox.layers.15.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2363392 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2365440 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2367488 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 2369536 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 3942400 }, { "name": "gpt_neox.layers.15.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 4139008 }, { "name": "gpt_neox.layers.15.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 4145152 }, { "name": "gpt_neox.layers.15.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 4669440 }, { "name": "gpt_neox.layers.15.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4734976 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 4737024 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 6834176 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 7096320 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7104512 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9201664 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9463808 }, { "name": "gpt_neox.layers.16.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9465856 }, { "name": "gpt_neox.layers.16.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9467904 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9469952 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 9472000 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 9474048 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 11046912 }, { "name": "gpt_neox.layers.16.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 11243520 }, { "name": "gpt_neox.layers.16.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 11249664 }, { "name": "gpt_neox.layers.16.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 11773952 }, { "name": "gpt_neox.layers.16.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11839488 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 11841536 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 13938688 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 14200832 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 14209024 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 16306176 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16568320 }, { "name": "gpt_neox.layers.17.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16570368 }, { "name": "gpt_neox.layers.17.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16572416 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16574464 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16576512 }, { "name": "gpt_neox.layers.17.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 16578560 }, { "name": "gpt_neox.layers.17.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 18151424 }, { "name": "gpt_neox.layers.17.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 18348032 }, { "name": "gpt_neox.layers.17.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 18354176 }, { "name": "gpt_neox.layers.17.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 18878464 }, { "name": "gpt_neox.layers.17.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18944000 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 18946048 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 21043200 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 21305344 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21313536 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 23410688 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23672832 }, { "name": "gpt_neox.layers.18.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23674880 }, { "name": "gpt_neox.layers.18.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23676928 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23678976 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 23681024 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 23683072 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 25255936 }, { "name": "gpt_neox.layers.18.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 25452544 }, { "name": "gpt_neox.layers.18.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 25458688 }, { "name": "gpt_neox.layers.18.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 25982976 }, { "name": "gpt_neox.layers.18.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26048512 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26050560 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28147712 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 28409856 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28418048 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 30515200 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30777344 }, { "name": "gpt_neox.layers.19.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30779392 }, { "name": "gpt_neox.layers.19.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30781440 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30783488 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 30785536 }, { "name": "gpt_neox.layers.19.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 30787584 }, { "name": "gpt_neox.layers.19.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 32360448 }, { "name": "gpt_neox.layers.19.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 32557056 }, { "name": "gpt_neox.layers.19.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 32563200 }, { "name": "gpt_neox.layers.19.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 33087488 }, { "name": "gpt_neox.layers.19.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33153024 } ], "md5sum": "9138a0950e3a35eb9051beedf39ba30a" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 25755648, "records": [ { "name": "embed_out.q_weight", "shape": [ 50304, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 25755648, "byteOffset": 0 } ], "md5sum": "69270e949e1cc6f7e240cd1c9377d5fa" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 33150976, "records": [ { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 0 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 2097152 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 2359296 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2367488 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 4464640 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4726784 }, { "name": "gpt_neox.layers.20.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4728832 }, { "name": "gpt_neox.layers.20.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4730880 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4732928 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4734976 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 4737024 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 6309888 }, { "name": "gpt_neox.layers.20.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6506496 }, { "name": "gpt_neox.layers.20.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 6512640 }, { "name": "gpt_neox.layers.20.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 7036928 }, { "name": "gpt_neox.layers.20.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 7102464 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7104512 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9201664 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 9463808 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 9472000 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11569152 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11831296 }, { "name": "gpt_neox.layers.21.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11833344 }, { "name": "gpt_neox.layers.21.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11835392 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11837440 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 11839488 }, { "name": "gpt_neox.layers.21.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 11841536 }, { "name": "gpt_neox.layers.21.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 13414400 }, { "name": "gpt_neox.layers.21.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 13611008 }, { "name": "gpt_neox.layers.21.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 13617152 }, { "name": "gpt_neox.layers.21.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 14141440 }, { "name": "gpt_neox.layers.21.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 14206976 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 14209024 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 16306176 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 16568320 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16576512 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18673664 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18935808 }, { "name": "gpt_neox.layers.22.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18937856 }, { "name": "gpt_neox.layers.22.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18939904 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18941952 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18944000 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 18946048 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 20518912 }, { "name": "gpt_neox.layers.22.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 20715520 }, { "name": "gpt_neox.layers.22.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 20721664 }, { "name": "gpt_neox.layers.22.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 21245952 }, { "name": "gpt_neox.layers.22.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 21311488 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21313536 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 23410688 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 23672832 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23681024 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 25778176 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26040320 }, { "name": "gpt_neox.layers.23.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26042368 }, { "name": "gpt_neox.layers.23.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26044416 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26046464 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 26048512 }, { "name": "gpt_neox.layers.23.attention.query_key_value.q_weight", "shape": [ 3072, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1572864, "byteOffset": 26050560 }, { "name": "gpt_neox.layers.23.attention.query_key_value.q_scale", "shape": [ 3072, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 196608, "byteOffset": 27623424 }, { "name": "gpt_neox.layers.23.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 27820032 }, { "name": "gpt_neox.layers.23.attention.dense.q_weight", "shape": [ 1024, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 27826176 }, { "name": "gpt_neox.layers.23.attention.dense.q_scale", "shape": [ 1024, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 28350464 }, { "name": "gpt_neox.layers.23.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 28416000 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight", "shape": [ 4096, 128 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28418048 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale", "shape": [ 4096, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 30515200 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 8192, "byteOffset": 30777344 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight", "shape": [ 1024, 512 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 30785536 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale", "shape": [ 1024, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 32882688 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33144832 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33146880 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 33148928 } ], "md5sum": "a81a6db3611f269bc5690557a6fa5877" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 3219456, "records": [ { "name": "embed_out.q_scale", "shape": [ 50304, 32 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3219456, "byteOffset": 0 } ], "md5sum": "ecc56611193223a982231f3f2ec7b649" } ] }