diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3235 @@ +{ + "metadata": { + "ParamSize": 292, + "ParamBytes": 810913792.0, + "BitsPerParam": 16.004850518146498 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 103022592, + "records": [ + { + "name": "gpt_neox.embed_in.weight", + "shape": [ + 50304, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 103022592, + "byteOffset": 0 + } + ], + "md5sum": "81ca3cd4cee0b0c05d93b0f0ed68c6fc" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 31498240, + "records": [ + { + "name": "gpt_neox.layers.0.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 2048 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 4096 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 6144 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 8192 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 6299648 + }, + { + "name": "gpt_neox.layers.0.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 6305792 + }, + { + "name": "gpt_neox.layers.0.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 8402944 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8404992 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16801792 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25192448 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25194496 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25196544 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25198592 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 25200640 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 31492096 + } + ], + "md5sum": "42bb31868b52ef9904b7a3622e1bbc70" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 27291648, + "records": [ + { + "name": "gpt_neox.layers.1.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.1.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 2097152 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 2099200 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 10487808 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 10496000 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 18884608 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 18886656 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 18888704 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 18890752 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 18892800 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 18894848 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 25186304 + }, + { + "name": "gpt_neox.layers.2.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 25192448 + }, + { + "name": "gpt_neox.layers.2.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 27289600 + } + ], + "md5sum": "e1416f837da59758eb1c1c6ea3e41659" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.3.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.3.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "94f6d168d09912f9759d9226c2e1241d" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.4.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.4.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "9551a45f3798632785e510aca673c043" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.5.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.5.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "319b288d5db6fde2bfdc6a028c9ca531" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.6.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.6.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "de3444038fd67975e2c2ab61b495e3d7" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.7.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.7.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "dcfacf8b737b9c9986cfebe5dbb2d21d" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.8.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.8.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "3b7f83152b7e14ef4e95dfe922d3dd77" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.9.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.9.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "be4ca56624464dcc9cb4acde88086432" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.10.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.10.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "162265ff489ae0ebe1084ad2f218a9d6" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.11.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.11.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "f000db5388c04a91711f01b3f936ddbf" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.12.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.12.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "5dc4f1f019c350656754d03e5be1b953" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.13.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.13.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "a18fea872b50f00f5bd397c85bb7a336" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.14.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.14.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "38f70a1d9119be8493fdc061b2746ab5" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.15.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.15.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "3de93f54324be9b8e16f84dda1fe601a" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.16.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.16.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "dfbb00066794f13962dbf7176f5965fe" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.17.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.17.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "b6719c0f9d35492b0f4ca81205673b24" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.18.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.18.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "71198570b0edcc3caa519f70d3da4a12" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.19.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.19.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "a28c9627ebb5d606dc9a854276379689" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.20.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.20.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "24df95d19a93b898e3c8e478e9b3809b" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.21.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.21.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "08edb389d5b33f336ac6c825e2fe7c48" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.22.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.22.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "dc3a4ab2fdd3ea4dddc7303156f88146" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 25192448, + "records": [ + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16791552 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16793600 + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.weight", + "shape": [ + 3072, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 16795648 + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.bias", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 23087104 + }, + { + "name": "gpt_neox.layers.23.attention.dense.weight", + "shape": [ + 1024, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 23093248 + }, + { + "name": "gpt_neox.layers.23.attention.dense.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 25190400 + } + ], + "md5sum": "7334140b022d9da47ad89b3692a02739" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 103022592, + "records": [ + { + "name": "embed_out.weight", + "shape": [ + 50304, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 103022592, + "byteOffset": 0 + } + ], + "md5sum": "d4e1ffb4064bb5794605cfce6c9d7d97" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 16791552, + "records": [ + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.weight", + "shape": [ + 4096, + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", + "shape": [ + 4096 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.weight", + "shape": [ + 1024, + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8396800 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", + "shape": [ + 1024 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16785408 + }, + { + "name": "gpt_neox.final_layer_norm.weight", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16787456 + }, + { + "name": "gpt_neox.final_layer_norm.bias", + "shape": [ + 1024 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2048, + "byteOffset": 16789504 + } + ], + "md5sum": "2012683990e73d3ff74c35bcf6860c72" + } + ] +} \ No newline at end of file