{ "metadata": { "ParamSize": 262, "ParamBytes": 570073088.0, "BitsPerParam": 4.507479242319355 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50304, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "7b085a504a46a91667729f1bf03a62ae" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 25362432, "records": [ { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50304, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6438912, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6438912 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6443008 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6447104 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6451200 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 6455296 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 12746752 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 13533184 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 13545472 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 15642624 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 15904768 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 15908864 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 24297472 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 25346048 } ], "md5sum": "71984128dad9eaf2831848474fb1586b" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "ed2fee48a5ebcc171d6574b2cd8b3e8e" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "d7133943be4e4be9bbbcda66823f6883" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "d7510f14df4e810ca4e0d74b4e8f461a" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "3d1df7886f065585c49345513703b14c" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "57606259e4125a002164fca22ea5ed80" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.6.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.6.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "5fa049d88339b7b67d44c38ddeeec037" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.7.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.7.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "f10dc2290035feb7fd3214d2bf3a6e96" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.8.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.8.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "cf089e45463d0ec256ebee581af36d20" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.9.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.9.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "192b75b00114d3eb4d664b58988e9ca8" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.10.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.10.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "b697bb21deb58369b464308d201e9e39" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.11.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.11.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "dee18a9b2d75b18fd2690aa5b69d9339" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.12.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.12.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.12.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.12.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.12.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.12.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "951c052e46cba0cd921ae0e24cd4b9fc" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.13.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.13.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.13.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.13.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.13.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.13.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "db87674ec77e107de16cb340532dd403" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.14.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.14.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.14.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.14.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.14.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.14.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "52f7da73759d96f0565985ec0a18f194" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 28364800, "records": [ { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.layers.15.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.layers.15.input_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9449472 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9453568 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", "shape": [ 6144, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 9457664 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", "shape": [ 6144, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 786432, "byteOffset": 15749120 }, { "name": "gpt_neox.layers.15.attention.query_key_value.bias", "shape": [ 6144 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 12288, "byteOffset": 16535552 }, { "name": "gpt_neox.layers.15.attention.dense.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16547840 }, { "name": "gpt_neox.layers.15.attention.dense.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 18644992 }, { "name": "gpt_neox.layers.15.attention.dense.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18907136 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", "shape": [ 8192, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18911232 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", "shape": [ 8192, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 27299840 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", "shape": [ 8192 ], "dtype": "bfloat16", "format": "raw", "nbytes": 16384, "byteOffset": 28348416 } ], "md5sum": "c7857fa0d998e5fd90e9d5c7d679e35c" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 51511296, "records": [ { "name": "embed_out.q_weight", "shape": [ 50304, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 51511296, "byteOffset": 0 } ], "md5sum": "46093692f024b0a331d0564858498deb" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 15888384, "records": [ { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", "shape": [ 2048, 1024 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", "shape": [ 2048, 256 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", "shape": [ 2048 ], "dtype": "bfloat16", "format": "raw", "nbytes": 4096, "byteOffset": 9437184 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9441280 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9445376 }, { "name": "embed_out.q_scale", "shape": [ 50304, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6438912, "byteOffset": 9449472 } ], "md5sum": "68926c29f381ae6c2b63baafc1b329e0" } ] }