Download ndarray-cache-b16.json from AMAI-Lab/pythia-410m: direct link, hf CLI and curl.
- Browser
- Download file 114 kB
-
https://huggingface.co/AMAI-Lab/pythia-410m/resolve/main/ndarray-cache-b16.json
- Command line
-
hf download hf://AMAI-Lab/pythia-410m/ndarray-cache-b16.json
-
curl -L -o ndarray-cache-b16.json https://huggingface.co/AMAI-Lab/pythia-410m/resolve/main/ndarray-cache-b16.json
114 kB
{
"metadata": {
"ParamSize": 292,
"ParamBytes": 810913792.0,
"BitsPerParam": 16.004850518146498
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 103022592,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 103022592,
"byteOffset": 0
}
],
"md5sum": "81ca3cd4cee0b0c05d93b0f0ed68c6fc"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 31498240,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2048
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4096
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 6144
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 8192
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 6299648
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 6305792
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 8402944
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8404992
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 16801792
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 25190400
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25192448
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25194496
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25196544
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25198592
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 25200640
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 31492096
}
],
"md5sum": "42bb31868b52ef9904b7a3622e1bbc70"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 27291648,
"records": [
{
"name": "gpt_neox.layers.1.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2097152
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 2099200
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 10487808
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 10496000
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 18884608
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18886656
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18888704
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18890752
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18892800
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 18894848
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 25186304
},
{
"name": "gpt_neox.layers.2.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 25192448
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 27289600
}
],
"md5sum": "e1416f837da59758eb1c1c6ea3e41659"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.3.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "94f6d168d09912f9759d9226c2e1241d"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.4.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "9551a45f3798632785e510aca673c043"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.5.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "319b288d5db6fde2bfdc6a028c9ca531"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.6.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "de3444038fd67975e2c2ab61b495e3d7"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.7.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "dcfacf8b737b9c9986cfebe5dbb2d21d"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.8.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "3b7f83152b7e14ef4e95dfe922d3dd77"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.9.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "be4ca56624464dcc9cb4acde88086432"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.10.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "162265ff489ae0ebe1084ad2f218a9d6"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.11.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "f000db5388c04a91711f01b3f936ddbf"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.12.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.12.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.12.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.12.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "5dc4f1f019c350656754d03e5be1b953"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.13.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.13.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.13.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.13.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "a18fea872b50f00f5bd397c85bb7a336"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.14.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.14.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.14.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.14.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "38f70a1d9119be8493fdc061b2746ab5"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.15.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.15.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.15.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.15.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "3de93f54324be9b8e16f84dda1fe601a"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.16.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.16.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.16.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.16.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "dfbb00066794f13962dbf7176f5965fe"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.17.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.17.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.17.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.17.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "b6719c0f9d35492b0f4ca81205673b24"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.18.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.18.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.18.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.18.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "71198570b0edcc3caa519f70d3da4a12"
},
{
"dataPath": "params_shard_19.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.19.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.19.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.19.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.19.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "a28c9627ebb5d606dc9a854276379689"
},
{
"dataPath": "params_shard_20.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.20.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.20.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.20.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.20.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "24df95d19a93b898e3c8e478e9b3809b"
},
{
"dataPath": "params_shard_21.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.21.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.21.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.21.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.21.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "08edb389d5b33f336ac6c825e2fe7c48"
},
{
"dataPath": "params_shard_22.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.22.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.22.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.22.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.22.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "dc3a4ab2fdd3ea4dddc7303156f88146"
},
{
"dataPath": "params_shard_23.bin",
"format": "raw-shard",
"nbytes": 25192448,
"records": [
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.layers.23.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.layers.23.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16791552
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16793600
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 16795648
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23087104
},
{
"name": "gpt_neox.layers.23.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23093248
},
{
"name": "gpt_neox.layers.23.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 25190400
}
],
"md5sum": "7334140b022d9da47ad89b3692a02739"
},
{
"dataPath": "params_shard_24.bin",
"format": "raw-shard",
"nbytes": 103022592,
"records": [
{
"name": "embed_out.weight",
"shape": [
50304,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 103022592,
"byteOffset": 0
}
],
"md5sum": "d4e1ffb4064bb5794605cfce6c9d7d97"
},
{
"dataPath": "params_shard_25.bin",
"format": "raw-shard",
"nbytes": 16791552,
"records": [
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.weight",
"shape": [
4096,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 8192,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.weight",
"shape": [
1024,
4096
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 8396800
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 2048,
"byteOffset": 16785408
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16787456
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16789504
}
],
"md5sum": "2012683990e73d3ff74c35bcf6860c72"
}
]
}
{
"metadata": {
"ParamSize": 292,
"ParamBytes": 810913792.0,
"BitsPerParam": 16.004850518146498
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 103022592,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 103022592,
"byteOffset": 0
}
],
"md5sum": "81ca3cd4cee0b0c05d93b0f0ed68c6fc"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 31498240,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2048
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4096
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 6144
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
3072,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 8192
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 6299648
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
1024,
1024
],
"dtype": "float16",
"format": "f32-to-bf16",