{ "metadata": { "ParamSize": 292, "ParamBytes": 810913792.0, "BitsPerParam": 16.004850518146498 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 103022592, "records": [ { "name": "gpt_neox.embed_in.weight", "shape": [ 50304, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 103022592, "byteOffset": 0 } ], "md5sum": "81ca3cd4cee0b0c05d93b0f0ed68c6fc" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 31498240, "records": [ { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2048 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 4096 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 6144 }, { "name": "gpt_neox.layers.0.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 8192 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 6299648 }, { "name": "gpt_neox.layers.0.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 6305792 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 8402944 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8404992 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 16801792 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 25190400 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25192448 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25194496 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25196544 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25198592 }, { "name": "gpt_neox.layers.1.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 25200640 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 31492096 } ], "md5sum": "42bb31868b52ef9904b7a3622e1bbc70" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 27291648, "records": [ { "name": "gpt_neox.layers.1.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 0 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 2097152 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 2099200 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 10487808 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10496000 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 18884608 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18886656 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18888704 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18890752 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 18892800 }, { "name": "gpt_neox.layers.2.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 18894848 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 25186304 }, { "name": "gpt_neox.layers.2.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 25192448 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 27289600 } ], "md5sum": "e1416f837da59758eb1c1c6ea3e41659" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.3.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.3.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "94f6d168d09912f9759d9226c2e1241d" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.4.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.4.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "9551a45f3798632785e510aca673c043" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.5.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.5.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "319b288d5db6fde2bfdc6a028c9ca531" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.6.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.6.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "de3444038fd67975e2c2ab61b495e3d7" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.7.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.7.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "dcfacf8b737b9c9986cfebe5dbb2d21d" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.8.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.8.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "3b7f83152b7e14ef4e95dfe922d3dd77" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.9.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.9.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "be4ca56624464dcc9cb4acde88086432" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.10.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.10.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "162265ff489ae0ebe1084ad2f218a9d6" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.11.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.11.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "f000db5388c04a91711f01b3f936ddbf" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.12.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.12.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.12.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.12.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.12.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.12.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "5dc4f1f019c350656754d03e5be1b953" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.13.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.13.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.13.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.13.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.13.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.13.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "a18fea872b50f00f5bd397c85bb7a336" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.14.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.14.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.14.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.14.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.14.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.14.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "38f70a1d9119be8493fdc061b2746ab5" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.15.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.15.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.15.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.15.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.15.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.15.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "3de93f54324be9b8e16f84dda1fe601a" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.16.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.16.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.16.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.16.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.16.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.16.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "dfbb00066794f13962dbf7176f5965fe" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.17.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.17.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.17.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.17.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.17.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.17.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "b6719c0f9d35492b0f4ca81205673b24" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.18.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.18.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.18.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.18.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.18.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.18.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "71198570b0edcc3caa519f70d3da4a12" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.19.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.19.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.19.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.19.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.19.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.19.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "a28c9627ebb5d606dc9a854276379689" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.20.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.20.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.20.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.20.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.20.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.20.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "24df95d19a93b898e3c8e478e9b3809b" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.21.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.21.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.21.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.21.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.21.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.21.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "08edb389d5b33f336ac6c825e2fe7c48" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.22.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.22.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.22.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.22.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.22.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.22.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "dc3a4ab2fdd3ea4dddc7303156f88146" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 25192448, "records": [ { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.layers.23.input_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.layers.23.input_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16791552 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16793600 }, { "name": "gpt_neox.layers.23.attention.query_key_value.weight", "shape": [ 3072, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6291456, "byteOffset": 16795648 }, { "name": "gpt_neox.layers.23.attention.query_key_value.bias", "shape": [ 3072 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 6144, "byteOffset": 23087104 }, { "name": "gpt_neox.layers.23.attention.dense.weight", "shape": [ 1024, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 23093248 }, { "name": "gpt_neox.layers.23.attention.dense.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 25190400 } ], "md5sum": "7334140b022d9da47ad89b3692a02739" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 103022592, "records": [ { "name": "embed_out.weight", "shape": [ 50304, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 103022592, "byteOffset": 0 } ], "md5sum": "d4e1ffb4064bb5794605cfce6c9d7d97" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 16791552, "records": [ { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.weight", "shape": [ 4096, 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", "shape": [ 4096 ], "dtype": "bfloat16", "format": "raw", "nbytes": 8192, "byteOffset": 8388608 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.weight", "shape": [ 1024, 4096 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 8396800 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", "shape": [ 1024 ], "dtype": "bfloat16", "format": "raw", "nbytes": 2048, "byteOffset": 16785408 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16787456 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 1024 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2048, "byteOffset": 16789504 } ], "md5sum": "2012683990e73d3ff74c35bcf6860c72" } ] }