diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3683 @@ +{ + "metadata": { + "ParamSize": 292, + "ParamBytes": 2829787136.0, + "BitsPerParam": 16.002779603501143 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 206045184, + "records": [ + { + "name": "gpt_neox.embed_in.weight", + "shape": [ + 50304, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 206045184, + "byteOffset": 0 + } + ], + "md5sum": "5c85d469d3a6358e6347e3d436bd31bf" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 25194496, + "records": [ + { + "name": "gpt_neox.layers.0.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 4096 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8192 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 12288 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 16384 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25182208 + } + ], + "md5sum": "e9a26a7d91f1dd5aa9ec5a31a81eaf10" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "e9f037184c8a59a774c5be6025e82767" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5e0386abd4410d1573e31ee499943b4d" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.1.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "cdc3abdd0a16c79ecc23b3556937a5dd" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "1418e5b945743e6a20885808db2f622c" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "ad1cb92ebcc2ce96282f3c00f58da63b" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.2.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6b8d7161b41b9fed0e4ba52d80d9ec24" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "3a5ab570c4ba3c5e0f33ec0c673e2493" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "f71aba1427bca21e1d29d61946990120" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.3.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3da95990b08e04ca83c188524f5768b5" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.0.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.1.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.1.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.2.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.2.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "ad5604a32091a27b4f37422beb5c7127" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "58b49fd1d5e3a8d05d9c8b9ee95a1c4b" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "1a1b8a67a671d406dd93492ddb815601" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.4.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "0392972a04f5ae29ff6621ca610dae21" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "2a8995d68b63ed87095c98aba7a0fd47" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "28471949d20439bca15dade58db44294" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.5.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "08271db9bdb1dd2c4c5bec8f957d7719" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5fa43ba0049345a1d2e8f7cf2bfb9748" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "670aa4e48da4b2d38001281fdbbc1be8" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.6.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c9fa78aca5c109288be40321e637f6d2" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.3.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.3.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.4.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.4.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.5.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.5.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "08321f9d815211578a56e0c10fd8b1e8" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "c961341e57b11112ec7e90cc514afe5a" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "17ceddba5f67f049981dcd8a99248205" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.7.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1a871777026dc7718f612cfa9609660f" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "024d1739808880b33539763db416004a" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "6f3e2d630526e4c3d78abed57e49bf87" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.8.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5f9351db9f5ab811bdcc91b70d61b6cf" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5245375f86c9d3ddaef4c0c6ba560545" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "3163ff001e468b2f96a4eaa806436693" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.9.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c149b7484226eb92bd7e762609ac9877" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.6.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.6.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.7.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.7.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.8.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.8.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "7005f9c34f17233961dfc6edebaad8eb" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "cd488d1027324116a2f63d7716938388" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "e18919f3e55dabe26f4a95bafcfc5a94" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.10.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "81ba98beff1928f2878726b897b5f50f" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "3fa173e12024e8f713959b35c584c347" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "91d488935d479fd044145ef160e622f0" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.11.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "4691957bfad6f60abb5ae21aa65afc4f" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "89032d5755b98354f1f51711f5c16605" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "8eaf9647301b564d89efd579dd7d0705" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.12.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "12de3c278d4729698b80bed6853dc94b" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.9.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.9.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.10.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.10.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.11.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.11.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "2d2c16e0f24eea73ed9c12ea2b48e044" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "bf44c2f3eea6c63a449002229e4f9b53" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "8109f192e82840af3d7115dbb17aa0a8" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.13.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "44f1043bb9c30fe23a1d8e747e6c5a1e" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "8acb05b2b698ef701894f2ba340b2923" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "c05dedd7a76c3755dbbecd0190fce1e2" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.14.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f7a17b48338f6ac417b432292d9760ae" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "a00ec52fc60d8b9f488a51121519d442" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "979b79166cc3ac7748525748ff9e847f" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.15.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "4705f8edbeb4285aecb78f2d0ca96394" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.12.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.12.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.13.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.13.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.14.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.14.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "501211c07a1bf63effb67a9e7362c947" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5dc9a03450b51fc089323e231ec6fca6" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "8b7f08d0cad3d23ef1919533b50b45a0" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.16.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "7380f2361468bad81bee85c71981bcfe" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "2e7e72f351414b0cdf813c051e1bc694" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "897b512dad54e52b28b30be80bfecd4a" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.17.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "da21123eb84a87b76b4907d34b7c24b8" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5b2bbf1b88d9c8aa5078f5680062f0b4" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "99872e9f912cd701b82243a90fa3775e" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.18.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "867e8116a0e289814e00f32d60cf3969" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.15.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.15.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.16.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.16.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.17.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.17.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "959b4278248634bc1d36bf764ee9eac2" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "aa8886f80b34d8f3bdc3d6d32caad484" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "ac2ec07fd034b463dc3c993e6a043271" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.19.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9c7564b9ff65091721b121fadba64574" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "62b820ae1b7727969671e615d66c5d1d" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "37235d7f21ffa06e87f10ae12a99fa77" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.20.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2bf4b95431e7071409cc84d078e6a65c" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "35a5ea1a59e74788df302d8503f592b1" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "5224dbd8c457753abe683ad6c2ea3b37" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.21.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c0a33934b7410702c8e3ae6a7f168e91" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 25325568, + "records": [ + { + "name": "gpt_neox.layers.18.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.18.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.19.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.19.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.20.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.20.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25305088 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25309184 + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 25313280 + } + ], + "md5sum": "fb78987924454d684bac1b3ac0d37829" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "da22921dcc6b389d67d8b16d33788c82" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "a94a9d1929172e22c2987dcda72606ad" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.22.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "36d5791b9f579063f3fa3e876a3f1da9" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "6c148b412af5327503f4febdc05c729a" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "016f8c0d05ec55e6d1d5b76ec18fce78" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "gpt_neox.layers.23.attention.query_key_value.weight", + "shape": [ + 6144, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "488de942e5bc384681f6db442f530b06" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.weight", + "shape": [ + 8192, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "f8fcc3142e6b55b263cff50916a9d367" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 33554432, + "records": [ + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.weight", + "shape": [ + 2048, + 8192 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 33554432, + "byteOffset": 0 + } + ], + "md5sum": "55f4840aaf65e171a34a7ec6d8024d88" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 206045184, + "records": [ + { + "name": "embed_out.weight", + "shape": [ + 50304, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 206045184, + "byteOffset": 0 + } + ], + "md5sum": "520be5b8c74b70dfe677b2e5cb922484" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 25305088, + "records": [ + { + "name": "gpt_neox.layers.21.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.21.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 8392704 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8409088 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8413184 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8417280 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8421376 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 8425472 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 8429568 + }, + { + "name": "gpt_neox.layers.22.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 8441856 + }, + { + "name": "gpt_neox.layers.22.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16830464 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 16834560 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16850944 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16855040 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16859136 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16863232 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 16867328 + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16871424 + }, + { + "name": "gpt_neox.layers.23.attention.dense.weight", + "shape": [ + 2048, + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16883712 + }, + { + "name": "gpt_neox.layers.23.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25272320 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25276416 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25292800 + }, + { + "name": "gpt_neox.final_layer_norm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25296896 + }, + { + "name": "gpt_neox.final_layer_norm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 25300992 + } + ], + "md5sum": "9d75f956fed0fb0147e95ca9db3156a3" + } + ] +} \ No newline at end of file