diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,2913 @@ +{ + "metadata": { + "ParamSize": 262, + "ParamBytes": 570073088.0, + "BitsPerParam": 4.507479242319355 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 51511296, + "records": [ + { + "name": "gpt_neox.embed_in.q_weight", + "shape": [ + 50304, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 51511296, + "byteOffset": 0 + } + ], + "md5sum": "7b085a504a46a91667729f1bf03a62ae" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 25362432, + "records": [ + { + "name": "gpt_neox.embed_in.q_scale", + "shape": [ + 50304, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6438912, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 6438912 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 6443008 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 6447104 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 6451200 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 6455296 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 12746752 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 13533184 + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 13545472 + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 15642624 + }, + { + "name": "gpt_neox.layers.0.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15904768 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 15908864 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 24297472 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 25346048 + } + ], + "md5sum": "71984128dad9eaf2831848474fb1586b" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.1.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "ed2fee48a5ebcc171d6574b2cd8b3e8e" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.2.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "d7133943be4e4be9bbbcda66823f6883" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.3.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "d7510f14df4e810ca4e0d74b4e8f461a" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.4.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "3d1df7886f065585c49345513703b14c" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.5.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "57606259e4125a002164fca22ea5ed80" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.6.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "5fa049d88339b7b67d44c38ddeeec037" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.7.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "f10dc2290035feb7fd3214d2bf3a6e96" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.8.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "cf089e45463d0ec256ebee581af36d20" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.9.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "192b75b00114d3eb4d664b58988e9ca8" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.10.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "b697bb21deb58369b464308d201e9e39" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.11.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "dee18a9b2d75b18fd2690aa5b69d9339" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.12.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "951c052e46cba0cd921ae0e24cd4b9fc" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.13.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "db87674ec77e107de16cb340532dd403" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.14.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "52f7da73759d96f0565985ec0a18f194" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 28364800, + "records": [ + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9449472 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9453568 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9457664 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 786432, + "byteOffset": 15749120 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.bias", + "shape": [ + 6144 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 12288, + "byteOffset": 16535552 + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16547840 + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 262144, + "byteOffset": 18644992 + }, + { + "name": "gpt_neox.layers.15.attention.dense.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 18907136 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18911232 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27299840 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", + "shape": [ + 8192 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 16384, + "byteOffset": 28348416 + } + ], + "md5sum": "c7857fa0d998e5fd90e9d5c7d679e35c" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 51511296, + "records": [ + { + "name": "embed_out.q_weight", + "shape": [ + 50304, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 51511296, + "byteOffset": 0 + } + ], + "md5sum": "46093692f024b0a331d0564858498deb" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 15888384, + "records": [ + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", + "shape": [ + 2048 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "gpt_neox.final_layer_norm.weight", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "gpt_neox.final_layer_norm.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "embed_out.q_scale", + "shape": [ + 50304, + 64 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6438912, + "byteOffset": 9449472 + } + ], + "md5sum": "68926c29f381ae6c2b63baafc1b329e0" + } + ] +} \ No newline at end of file