{ "metadata": { "ParamSize": 390, "ParamBytes": 797155328.0, "BitsPerParam": 4.508007284877509 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50304, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "16fdebd3852a007cb7c34043ca16f3b7" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 25362432, "records": [ { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50304, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6438912, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6438912 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6443008 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6447104 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6451200 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 6455296 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 12746752 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 13533184 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 13545472 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 15642624 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 15904768 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 15908864 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 24297472 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 25346048 } ], "md5sum": "e644c52d08710fd0fe00547ef61d82e7" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "4b3eda81e5416ace10022c6cc473b812" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "e672144a00cf3012ba468d4df29e8f12" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "c6450a0bd469417df70e0350c172a854" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "f850f3eed647be0f696e3cf8332286ea" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "268cbb136f0ff6952dc04eda673b8993" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.6.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.6.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "64822e98bb4213e94b70930a863b693c" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.7.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.7.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "7d53616d6eb0105f4e1f6e4a0a5a9376" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.8.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.8.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "cdf77d22f86088bf8754d171ff307140" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.9.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.9.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "723f81a6608b10d734237eebb1446917" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.10.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.10.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "a5476a576d4965310e8661e604ac4a5c" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.11.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.11.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "dfb67a58fac00474f53fa69ad177b8f4" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.12.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.12.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.12.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.12.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.12.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.12.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "9498a44233c610e36e7250c50c8f61b3" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.13.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.13.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.13.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.13.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.13.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.13.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "c9862d556db880d033bb5f8a643900ef" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.14.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.14.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.14.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.14.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.14.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.14.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "ea2d5fd621a165977f5dbe52558176a7" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.15.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.15.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.15.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.15.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.15.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.15.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "16fa48eb420e5e5d3f0dc64e610d9dff" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.16.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.16.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.16.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.16.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.16.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.16.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "75f293533e31161a0b03e26c34763b51" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.17.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.17.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.17.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.17.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.17.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.17.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.17.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.17.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "63d1676ab86d9fa88dfbd782c92711f5" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.18.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.18.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.18.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.18.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.18.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.18.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "a4e8fa391c91a7d1a0455dccfca0a7d6" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.19.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.19.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.19.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.19.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.19.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.19.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.19.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.19.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "2d0ba72978d03944d98fdd587fd85848" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.20.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.20.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.20.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.20.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.20.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.20.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "cfae96bc77e91b5227200668e237c087" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.21.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.21.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.21.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.21.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.21.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.21.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.21.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.21.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "88ba0a5b3ec12c6d208f819ffd6a3759" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.22.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.22.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.22.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.22.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.22.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.22.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "d4407c080a1ee8f14b7c240782731432" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.23.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.23.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.23.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.23.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.23.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.23.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.23.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.23.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "41996118f9f61fd36500d8535e0f6a80" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "embed_out.q_weight", "shape": [ 50304, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "852483dea0be9275e0ae266bdb041ac0" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 15888384, "records": [ { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "embed_out.q_scale", "shape": [ 50304, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6438912, "byteOffset": 9449472 } ], "md5sum": "a58aaaeb2e12e555007dcaa46f4e9a0d" } ] }