{ "metadata": { "ParamSize": 102, "ParamBytes": 39704576.0, "BitsPerParam": 4.510177969058974 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 25186304, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50304, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12877824, "byteOffset": 0 }, { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50304, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1609728, "byteOffset": 12877824 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 14487552 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 14488576 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 14489600 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 14490624 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 14491648 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 14884864 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 14934016 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 14937088 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 15068160 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 15084544 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 15085568 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 15609856 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 15675392 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 15679488 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 16203776 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16269312 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16270336 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16271360 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16272384 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16273408 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 16274432 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 16667648 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 16716800 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 16719872 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 16850944 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 16867328 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 16868352 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 17392640 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 17458176 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 17462272 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 17986560 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18052096 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18053120 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18054144 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18055168 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18056192 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 18057216 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 18450432 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 18499584 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 18502656 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 18633728 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 18650112 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 18651136 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 19175424 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 19240960 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 19245056 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 19769344 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 19834880 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 19835904 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 19836928 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 19837952 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 19838976 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 19840000 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 20233216 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 20282368 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 20285440 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 20416512 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 20432896 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 20433920 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 20958208 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21023744 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 21027840 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 21552128 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21617664 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21618688 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21619712 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21620736 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21621760 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 21622784 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 22016000 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 22065152 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 22068224 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 22199296 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 22215680 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 22216704 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 22740992 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 22806528 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 22810624 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 23334912 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23400448 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23401472 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23402496 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23403520 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23404544 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 1536, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 23405568 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 1536, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 49152, "byteOffset": 23798784 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 1536 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 3072, "byteOffset": 23847936 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 512, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 23851008 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 512, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 16384, "byteOffset": 23982080 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 23998464 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 2048, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 23999488 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 2048, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 24523776 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 2048 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 24589312 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 512, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 524288, "byteOffset": 24593408 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 512, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 65536, "byteOffset": 25117696 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 512 ], "dtype": "float32", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25183232 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25184256 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 25185280 } ], "md5sum": "3ec2318c60fdf10dd957b42f6ba5687a" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 14487552, "records": [ { "name": "embed_out.q_weight", "shape": [ 50304, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 12877824, "byteOffset": 0 }, { "name": "embed_out.q_scale", "shape": [ 50304, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1609728, "byteOffset": 12877824 } ], "md5sum": "fcbf5286e19627c8278b745710f4b86e" } ] }