Download ndarray-cache.json from AMAI-Lab/pythia-1b-q4f16-1: direct link, hf CLI and curl.
- Browser
- Download file 103 kB
-
https://huggingface.co/AMAI-Lab/pythia-1b-q4f16-1/resolve/main/ndarray-cache.json
- Command line
-
hf download hf://AMAI-Lab/pythia-1b-q4f16-1/ndarray-cache.json
-
curl -L -o ndarray-cache.json https://huggingface.co/AMAI-Lab/pythia-1b-q4f16-1/resolve/main/ndarray-cache.json
103 kB
{
"metadata": {
"ParamSize": 262,
"ParamBytes": 570073088.0,
"BitsPerParam": 4.507479242319355
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "7b085a504a46a91667729f1bf03a62ae"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 25362432,
"records": [
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6438912,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6438912
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6443008
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6447104
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6451200
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 6455296
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 12746752
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 13533184
},
{
"name": "gpt_neox.layers.0.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 13545472
},
{
"name": "gpt_neox.layers.0.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 15642624
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 15904768
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 15908864
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 24297472
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 25346048
}
],
"md5sum": "71984128dad9eaf2831848474fb1586b"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.1.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.1.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "ed2fee48a5ebcc171d6574b2cd8b3e8e"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.2.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.2.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "d7133943be4e4be9bbbcda66823f6883"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.3.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.3.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "d7510f14df4e810ca4e0d74b4e8f461a"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.4.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.4.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "3d1df7886f065585c49345513703b14c"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.5.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.5.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "57606259e4125a002164fca22ea5ed80"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.6.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.6.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "5fa049d88339b7b67d44c38ddeeec037"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.7.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.7.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "f10dc2290035feb7fd3214d2bf3a6e96"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.8.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.8.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "cf089e45463d0ec256ebee581af36d20"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.9.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.9.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "192b75b00114d3eb4d664b58988e9ca8"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.10.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.10.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "b697bb21deb58369b464308d201e9e39"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.11.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.11.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "dee18a9b2d75b18fd2690aa5b69d9339"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.12.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.12.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.12.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.12.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.12.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "951c052e46cba0cd921ae0e24cd4b9fc"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.13.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.13.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.13.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.13.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.13.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "db87674ec77e107de16cb340532dd403"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.14.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.14.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.14.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.14.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.14.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "52f7da73759d96f0565985ec0a18f194"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.15.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.15.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.15.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.15.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.15.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "c7857fa0d998e5fd90e9d5c7d679e35c"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "embed_out.q_weight",
"shape": [
50304,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "46093692f024b0a331d0564858498deb"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 15888384,
"records": [
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "embed_out.q_scale",
"shape": [
50304,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6438912,
"byteOffset": 9449472
}
],
"md5sum": "68926c29f381ae6c2b63baafc1b329e0"
}
]
}
{
"metadata": {
"ParamSize": 262,
"ParamBytes": 570073088.0,
"BitsPerParam": 4.507479242319355
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "7b085a504a46a91667729f1bf03a62ae"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 25362432,
"records": [
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6438912,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6438912
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6443008
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6447104
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6451200
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 6455296
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",