{ "metadata": { "ParamSize": 198, "ParamBytes": 91573248.0, "BitsPerParam": 4.513138844992856 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 32406528, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 19316736, "byteOffset": 0 }, { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2414592, "byteOffset": 19316736 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21731328 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21732864 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21734400 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21735936 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 21737472 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 22622208 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 22732800 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 22737408 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 23032320 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 23069184 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 23070720 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 24250368 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 24397824 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 24403968 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 25583616 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 25731072 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25732608 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25734144 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25735680 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25737216 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 25738752 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 26623488 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 26734080 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 26738688 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 27033600 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 27070464 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 27072000 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 28251648 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 28399104 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 28405248 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 29584896 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 29732352 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29733888 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29735424 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29736960 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29738496 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 29740032 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 30624768 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 30735360 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 30739968 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 31034880 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 31071744 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 31073280 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 32252928 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 32400384 } ], "md5sum": "f0227163eab2db5bdea0284280f0f447" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 33345024, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 1179648 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 1327104 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1328640 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1330176 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1331712 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1333248 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 1334784 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 2219520 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 2330112 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 2334720 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 2629632 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 2666496 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 2668032 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 3847680 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 3995136 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 4001280 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 5180928 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 5328384 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 5329920 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 5331456 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 5332992 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 5334528 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 5336064 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 6220800 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 6331392 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 6336000 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 6630912 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 6667776 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 6669312 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 7848960 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 7996416 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 8002560 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 9182208 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 9329664 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9331200 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9332736 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9334272 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 9335808 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 9337344 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 10222080 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 10332672 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 10337280 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 10632192 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 10669056 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 10670592 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 11850240 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 11997696 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 12003840 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 13183488 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 13330944 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 13332480 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 13334016 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 13335552 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 13337088 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 13338624 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 14223360 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 14333952 }, { "name": "gpt_neox.layers.6.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 14338560 }, { "name": "gpt_neox.layers.6.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 14633472 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 14670336 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 14671872 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 15851520 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 15998976 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 16005120 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 17184768 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 17332224 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 17333760 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 17335296 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 17336832 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 17338368 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 17339904 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 18224640 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 18335232 }, { "name": "gpt_neox.layers.7.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 18339840 }, { "name": "gpt_neox.layers.7.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 18634752 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 18671616 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 18673152 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 19852800 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 20000256 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 20006400 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 21186048 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 21333504 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21335040 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21336576 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21338112 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 21339648 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 21341184 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 22225920 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 22336512 }, { "name": "gpt_neox.layers.8.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 22341120 }, { "name": "gpt_neox.layers.8.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 22636032 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 22672896 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 22674432 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 23854080 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 24001536 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 24007680 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 25187328 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 25334784 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25336320 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25337856 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25339392 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 25340928 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 25342464 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 26227200 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 26337792 }, { "name": "gpt_neox.layers.9.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 26342400 }, { "name": "gpt_neox.layers.9.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 26637312 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 26674176 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 26675712 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 27855360 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 28002816 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 28008960 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 29188608 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 29336064 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29337600 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29339136 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29340672 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 29342208 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 29343744 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 30228480 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 30339072 }, { "name": "gpt_neox.layers.10.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 30343680 }, { "name": "gpt_neox.layers.10.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 30638592 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 30675456 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 30676992 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 31856640 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 32004096 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 32010240 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 33189888 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 33337344 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33338880 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33340416 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33341952 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 33343488 } ], "md5sum": "941fc9dffde95be24bd0b5d3febbbe15" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 25729536, "records": [ { "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", "shape": [ 2304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 884736, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", "shape": [ 2304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 110592, "byteOffset": 884736 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 2304 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4608, "byteOffset": 995328 }, { "name": "gpt_neox.layers.11.attention.dense.q_weight", "shape": [ 768, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 294912, "byteOffset": 999936 }, { "name": "gpt_neox.layers.11.attention.dense.q_scale", "shape": [ 768, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 36864, "byteOffset": 1294848 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 1331712 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", "shape": [ 3072, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 1333248 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", "shape": [ 3072, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 2512896 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 3072 ], "dtype": "bfloat16", "format": "raw", "nbytes": 6144, "byteOffset": 2660352 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", "shape": [ 768, 384 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1179648, "byteOffset": 2666496 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", "shape": [ 768, 96 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 147456, "byteOffset": 3846144 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 768 ], "dtype": "bfloat16", "format": "raw", "nbytes": 1536, "byteOffset": 3993600 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 3995136 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 768 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1536, "byteOffset": 3996672 }, { "name": "embed_out.q_weight", "shape": [ 50304, 96 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 19316736, "byteOffset": 3998208 }, { "name": "embed_out.q_scale", "shape": [ 50304, 24 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2414592, "byteOffset": 23314944 } ], "md5sum": "8cf5b4f860269d72cf81ee713307e594" } ] }